diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index af187705e13..c7c8b5de7b2 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -245,6 +245,26 @@ jobs: lib/knowledge/__integration__/connector-upload.integration.ts lib/uploads/contexts/organization-logo/application.integration.ts + - name: Verify Confluence identity and directory sync in PostgreSQL + working-directory: apps/sim + env: + KNOWLEDGE_ACL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim + run: >- + bunx vitest run --mode integration + lib/knowledge/__integration__/confluence-identity.integration.ts + lib/knowledge/__integration__/directory-sync.integration.ts + + - name: Verify Confluence audience migrations and permission queries in PostgreSQL + working-directory: apps/sim + env: + KNOWLEDGE_ACL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_acl_test + run: | + bun -e 'import postgres from "postgres"; const sql = postgres(process.env.DATABASE_URL); await sql`CREATE DATABASE sim_acl_test`; await sql.end()' + bunx vitest run --mode integration lib/knowledge/access/group-membership.integration.ts + bunx vitest run \ + lib/knowledge/access/predicate.postgres.test.ts \ + lib/knowledge/connectors/external-directory.postgres.test.ts + test-build: name: Lint and Test runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} diff --git a/apps/docs/content/docs/knowledgebase/connectors.mdx b/apps/docs/content/docs/knowledgebase/connectors.mdx index e3624676668..8aade70fc02 100644 --- a/apps/docs/content/docs/knowledgebase/connectors.mdx +++ b/apps/docs/content/docs/knowledgebase/connectors.mdx @@ -78,7 +78,7 @@ Each connector has source-specific fields that control what gets synced. Example - **Notion** — sync an entire workspace, a specific database, or a single page tree - **GitHub** — specify a repository, branch, and optional file extension filter -- **Confluence** — enter your Atlassian domain and optionally filter by space key or content type +- **Confluence** — enter your Atlassian domain and choose spaces, or **All** for all spaces accessible at each sync. Optionally filter by content type or label. PDF and Word (`.docx`, Word 97–2003 `.doc`) attachments on matching pages and blog posts are included as separate documents. - **Azure DevOps** — choose what to sync (wiki pages, work items, repository files, or all), with optional work item type/state filters, a custom WIQL query, and repository/branch/path filters - **Amazon S3** — point at a bucket with an optional key prefix and a customizable file extension allowlist; S3-compatible stores (Cloudflare R2, MinIO) are supported via a custom endpoint - **YouTube** — sync a channel (by `@handle` or ID) or playlist, with an optional published-after date filter and the option to exclude Shorts @@ -88,6 +88,8 @@ Each connector has source-specific fields that control what gets synced. Example Configuration is validated on save — if a repository doesn't exist or a domain is unreachable, you'll see an error immediately. +Confluence attachment indexing requires `read:attachment:confluence`. For a service account, include it when creating the scoped API token; see the [Confluence scope list](/search/confluence#using-a-service-account). Attachments are checked even when the parent page has not changed. Files over 100 MB appear as skipped; convert Word 6/95 files to `.docx` before attaching them. + diff --git a/apps/docs/content/docs/search/confluence.mdx b/apps/docs/content/docs/search/confluence.mdx index d07af6163bf..aab0393b058 100644 --- a/apps/docs/content/docs/search/confluence.mdx +++ b/apps/docs/content/docs/search/confluence.mdx @@ -7,7 +7,7 @@ import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -Search pages and blog posts from selected Confluence Cloud spaces. A Sim organization admin enables Confluence; **each teammate connects their own account**. +Search pages, blog posts, and their PDF and Word attachments from selected Confluence Cloud spaces. A Sim organization admin enables Confluence; **each teammate connects their own account**. | Method | How it works | | --- | --- | @@ -38,7 +38,9 @@ Open **Settings → Sources → Add source** and select **Confluence**. This ope ### Choose the account and spaces -Under **Service account**, select a service account or [add one](#using-a-service-account). Enter the same **Confluence site** as the credential, then choose **Spaces**. **All** in the dropdown selects every space the account can currently browse; newly created spaces are not added automatically. Clear the picker search before selecting all. +Under **Service account**, select a service account or [add one](#using-a-service-account). Enter the same **Confluence site** as the credential, then choose **Spaces**. **All** in the dropdown includes every space the syncing account can access at each sync, including newly accessible spaces. Clear the picker search before selecting all. + +If you selected all spaces before this behavior was introduced, reselect **All** and save. Previously saved selections remain a fixed list of spaces. To enter comma-separated keys such as `ENG, PRODUCT`, use the switch beside **Spaces**. Switching between the picker and manual entry keeps your selection. @@ -63,7 +65,7 @@ After an admin configures Confluence, open **Integrations** and select **Connect If Confluence is allowed but no source exists, select **Connect** beside Confluence. To add another site later, open the Confluence row’s actions menu (**…**) and select **Add Confluence site**: 1. Open **Your account** and select a saved account or **Connect Confluence account**. Authorize using the Atlassian email matching your verified Sim email. -2. Enter the hostname under **Atlassian site**, then choose **Spaces**. Use **All** in the dropdown for the complete current list, or the arrows beside **Spaces** to enter comma-separated keys. You can select up to 1,000 spaces in this form. +2. Enter the hostname under **Atlassian site**, then choose **Spaces**. Use **All** in the dropdown for all spaces accessible at each sync, or the arrows beside **Spaces** to enter comma-separated keys. You can select up to 1,000 individual spaces in this form. 3. Select **Connect & Sync**. Sim saves the selected scope and starts indexing with your account. ({ import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field' interface ComboboxCallbacks { - options: { value: string; label: string; hidden?: boolean; onSelect?: () => void }[] + options: { + value: string + label: string + hidden?: boolean + selected?: boolean + onSelect?: () => void + }[] disabled: boolean onChange?: (value: string) => void onMultiSelectChange?: (value: string[]) => void } +it('stores dynamic All without enumerating a snapshot and lets users select individual items again', async () => { + const field = { + id: 'spaces', + title: 'Spaces', + type: 'selector', + selectorKey: 'confluence.spaces', + multi: true, + allowSelectAll: true, + selectAllValue: '*', + } as const + const root = createRoot(document.createElement('div')) + const render = async (value: string[]) => + act(async () => + root.render( + + ) + ) + try { + await render([]) + await act(async () => + mocks.combobox.mock + .lastCall![0].options.find((option) => option.label === 'All') + ?.onSelect?.() + ) + expect(mocks.change).toHaveBeenLastCalledWith(['*'], [{ id: '*', label: 'All' }]) + expect(mocks.loadAll).not.toHaveBeenCalled() + await render(['*']) + await act(async () => mocks.combobox.mock.lastCall![0].onMultiSelectChange?.(['*', 'folder-b'])) + expect(mocks.change).toHaveBeenLastCalledWith( + ['folder-b'], + [{ id: 'folder-b', label: 'Company docs' }] + ) + await render(['*', 'folder-b']) + const all = mocks.combobox.mock.lastCall![0].options.find((option) => option.label === 'All') + await act(async () => all?.onSelect?.()) + expect(mocks.change).toHaveBeenLastCalledWith(['*'], [{ id: '*', label: 'All' }]) + } finally { + await act(async () => root.unmount()) + vi.clearAllMocks() + } +}) + beforeEach(() => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx index bdd170f7e50..eae49ed34e5 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx @@ -139,8 +139,11 @@ export function ConnectorSelectorField({ const singleValue = Array.isArray(value) ? value[0] : value const selectedIds = useMemo( - () => (Array.isArray(value) ? value : value ? [value] : []).filter(Boolean), - [value] + () => + (Array.isArray(value) ? value : value ? [value] : []).filter( + (id) => Boolean(id) && id !== field.selectAllValue + ), + [value, field.selectAllValue] ) const missingSelectedIds = useMemo(() => { const loadedIds = new Set(options.map((option) => option.id)) @@ -197,6 +200,9 @@ export function ConnectorSelectorField({ }, [options, selectedOptions, searchedOption, selectedLabels, selectedIds]) const handleChange = (nextValue: ConfigFieldValue) => { + if (Array.isArray(nextValue) && field.selectAllValue) { + nextValue = nextValue.filter((id) => id !== field.selectAllValue) + } bulkGenerationRef.current += 1 setBulkError(null) const ids = new Set(Array.isArray(nextValue) ? nextValue : nextValue ? [nextValue] : []) @@ -214,18 +220,26 @@ export function ConnectorSelectorField({ const hasSearch = searchTerm.trim().length > 0 || debouncedSearch.length > 0 const selectedIdSet = new Set(selectedIds) - const allSelected = - !hasMore && - !truncated && - options.length > 0 && - selectedIds.length === options.length && - options.every((option) => selectedIdSet.has(option.id)) + const values = Array.isArray(value) ? value : [value] + const allSelected = field.selectAllValue + ? values.length === 1 && values[0] === field.selectAllValue + : !hasMore && + !truncated && + options.length > 0 && + selectedIds.length === options.length && + options.every((option) => selectedIdSet.has(option.id)) const selectAll = async () => { if (!isEnabled || hasSearch || isFetching || isLoadingAll) return if (allSelected) { handleChange([]) return } + if (field.selectAllValue) { + bulkGenerationRef.current += 1 + setBulkError(null) + onChange([field.selectAllValue], [{ id: field.selectAllValue, label: 'All' }]) + return + } const generation = ++bulkGenerationRef.current setBulkError(null) const result = await loadAll() @@ -265,10 +279,10 @@ export function ConnectorSelectorField({ aria-label={field.title} multiSelect options={ - field.allowSelectAll && (options.length > 0 || hasMore) + field.allowSelectAll && (options.length > 0 || hasMore || allSelected) ? [ { - value: '', + value: field.selectAllValue ?? '', label: 'All', disabled: !isEnabled || hasSearch || isFetching || isLoadingAll, onSelect: () => void selectAll(), diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.ui.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.ui.test.tsx index 2f966afe39d..e3595c5094c 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.ui.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.ui.test.tsx @@ -39,7 +39,11 @@ vi.mock('@/hooks/queries/selectors', () => ({ import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field' -function ControlledSelector() { +interface ControlledSelectorProps { + dynamicAll?: boolean +} + +function ControlledSelector({ dynamicAll = false }: ControlledSelectorProps) { const [value, setValue] = useState([]) return ( { mocks.error = null }) -it('selects all pages with the keyboard, announces selection, and toggles it off', async () => { - vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) - const originalScroll = HTMLElement.prototype.scrollIntoView - HTMLElement.prototype.scrollIntoView = vi.fn() - mocks.loadAll.mockResolvedValue({ - status: 'complete', - options: [ - { id: 'ENG', label: 'Engineering' }, - { id: 'OPS', label: 'Operations' }, - ], - }) - const container = document.createElement('div') - document.body.appendChild(container) - const root = createRoot(container) - try { - await act(async () => root.render()) - expect(container.textContent).not.toContain('Select all') - expect(container.textContent).not.toContain('Clear') - const trigger = container.querySelector('[role="combobox"]') - expect(trigger).not.toBeNull() - await act(async () => - trigger?.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown', bubbles: true })) - ) - expect(document.querySelector('[role="option"]')?.textContent).toBe('All') - await act(async () => - trigger?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) - ) - expect(document.querySelector('[role="option"]')?.getAttribute('aria-selected')).toBe('true') - expect(mocks.loadAll).toHaveBeenCalledOnce() - expect(mocks.change).toHaveBeenCalledWith( - ['ENG', 'OPS'], - [ +it.each([false, true])( + 'selects All with the keyboard and toggles it off (dynamic: %s)', + async (dynamicAll) => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + const originalScroll = HTMLElement.prototype.scrollIntoView + HTMLElement.prototype.scrollIntoView = vi.fn() + mocks.loadAll.mockResolvedValue({ + status: 'complete', + options: [ { id: 'ENG', label: 'Engineering' }, { id: 'OPS', label: 'Operations' }, - ] - ) - await act(async () => - document - .querySelector('[role="option"]') - ?.dispatchEvent(new MouseEvent('mousedown', { bubbles: true })) - ) - expect(mocks.change).toHaveBeenLastCalledWith([], []) - expect(document.querySelector('[role="option"]')?.getAttribute('aria-selected')).toBe('false') - expect(mocks.loadAll).toHaveBeenCalledOnce() - } finally { - await act(async () => root.unmount()) - container.remove() - HTMLElement.prototype.scrollIntoView = originalScroll - vi.unstubAllGlobals() + ], + }) + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + try { + await act(async () => root.render()) + expect(container.textContent).not.toContain('Select all') + expect(container.textContent).not.toContain('Clear') + const trigger = container.querySelector('[role="combobox"]') + expect(trigger).not.toBeNull() + await act(async () => + trigger?.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown', bubbles: true })) + ) + expect(document.querySelector('[role="option"]')?.textContent).toBe('All') + await act(async () => + trigger?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + ) + expect(document.querySelector('[role="option"]')?.getAttribute('aria-selected')).toBe('true') + expect(mocks.loadAll).toHaveBeenCalledTimes(dynamicAll ? 0 : 1) + expect(mocks.change).toHaveBeenCalledWith( + dynamicAll ? ['*'] : ['ENG', 'OPS'], + dynamicAll + ? [{ id: '*', label: 'All' }] + : [ + { id: 'ENG', label: 'Engineering' }, + { id: 'OPS', label: 'Operations' }, + ] + ) + if (dynamicAll) expect(trigger?.textContent).toContain('All') + await act(async () => + document + .querySelector('[role="option"]') + ?.dispatchEvent(new MouseEvent('mousedown', { bubbles: true })) + ) + expect(mocks.change).toHaveBeenLastCalledWith([], []) + expect(document.querySelector('[role="option"]')?.getAttribute('aria-selected')).toBe('false') + expect(mocks.loadAll).toHaveBeenCalledTimes(dynamicAll ? 0 : 1) + } finally { + await act(async () => root.unmount()) + container.remove() + HTMLElement.prototype.scrollIntoView = originalScroll + vi.unstubAllGlobals() + } } -}) +) it.each(['empty', 'error'] as const)( 'shows the initial %s state without All and allows failed lists to retry', diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.test.tsx index fef39a83418..98e1067296b 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.test.tsx @@ -311,6 +311,40 @@ describe('useConnectorConfigFields member configuration', () => { } ) + it.each([ + [jiraConnectorMeta, 'projectKey', 'projectSelector'], + [confluenceConnectorMeta, 'spaceKey', 'spaceSelector'], + ] as const)( + 'preserves saved $0.name All scope through manual entry and clears it when the site changes', + (meta, canonicalId, selectorId) => { + render({ + connectorConfig: meta, + accessMode: 'members', + initialSourceConfig: { domain: 'team.atlassian.net', [selectorId]: ['*'] }, + initialSelectionLabels: { [canonicalId]: [{ id: '*', label: 'All' }] }, + }) + expect(current.resolveSourceConfig()[canonicalId]).toEqual(['*']) + act(() => current.toggleCanonicalMode(canonicalId)) + expect(current.sourceConfig[canonicalId]).toEqual(['*']) + expect(current.resolveSourceConfig()[canonicalId]).toEqual(['*']) + act(() => current.toggleCanonicalMode(canonicalId)) + expect(current.sourceConfig[selectorId]).toEqual(['*']) + expect(describeSearchSource(meta, current.resolveSourceConfig())).toBe( + 'team.atlassian.net · All' + ) + act(() => current.toggleCanonicalMode(canonicalId)) + act(() => current.handleFieldChange(canonicalId, 'ENG, PRODUCT')) + act(() => current.toggleCanonicalMode(canonicalId)) + expect(current.resolveSourceConfig()[canonicalId]).toEqual(['ENG', 'PRODUCT']) + act(() => current.handleFieldChange(selectorId, ['*'], [{ id: '*', label: 'All' }])) + act(() => current.handleFieldChange('domain', 'other.atlassian.net')) + expect(current.resolveSourceConfig()[canonicalId]).toEqual([]) + act(() => current.toggleCanonicalMode(canonicalId)) + expect(current.resolveSourceConfig()[canonicalId]).toEqual([]) + expect(current.selectionLabels).toEqual({}) + } + ) + it('clears dependent selector labels together with their values', () => { const meta: ConnectorMeta = { ...googleDriveConnectorMeta, diff --git a/apps/sim/connectors/all-selection.test.ts b/apps/sim/connectors/all-selection.test.ts new file mode 100644 index 00000000000..60711647332 --- /dev/null +++ b/apps/sim/connectors/all-selection.test.ts @@ -0,0 +1,158 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { clearAtlassianCloudIdCache } from '@/lib/atlassian/discovery' +import { confluenceConnector } from '@/connectors/confluence/confluence' +import { jiraConnector } from '@/connectors/jira/jira' + +const DOMAIN = 'all-validation.atlassian.net' +const CLOUD_ID = 'all-validation-cloud' +const fetchMock = vi.fn() + +beforeEach(() => { + fetchMock.mockReset() + clearAtlassianCloudIdCache() + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => vi.unstubAllGlobals()) + +describe('dynamic All source validation', () => { + describe.each([ + { name: 'Jira', connector: jiraConnector, field: 'projectKey' }, + { name: 'Confluence', connector: confluenceConnector, field: 'spaceKey' }, + ])('$name mixed All selection', ({ connector, field }) => { + const error = 'Use "*" by itself for All, or remove it to select individual items.' + + it.each([{ value: '*, ENG' }, { value: ['*', 'ENG'] }, { value: ['ENG', ' * '] }])( + 'rejects mixed keys before validation requests (%j)', + async ({ value }) => { + await expect( + connector.validateConfig('token', { domain: DOMAIN, [field]: value }) + ).resolves.toEqual({ valid: false, error }) + expect(fetchMock).not.toHaveBeenCalled() + } + ) + + it.each([false, true])( + 'rejects mixed keys before listing or splitting the scope (per member: %s)', + async (perMemberListing) => { + await expect( + connector.listDocuments('token', { domain: DOMAIN, [field]: ['*', 'ENG'] }, undefined, { + perMemberListing, + }) + ).rejects.toThrow(error) + expect(fetchMock).not.toHaveBeenCalled() + } + ) + + it('rejects mixed keys during resumed hydration before reading the provider', async () => { + await expect( + connector.getDocument('token', { domain: DOMAIN, [field]: 'ENG, *' }, 'document-1') + ).rejects.toThrow(error) + expect(fetchMock).not.toHaveBeenCalled() + }) + }) + + it.each([{ value: '*' }, { value: ['*'] }])( + 'validates Confluence All without enumerating or submitting literal space keys (%j)', + async ({ value: spaceKey }) => { + fetchMock.mockResolvedValueOnce( + Response.json({ + results: [{ id: 'space-1', key: 'ENG' }], + _links: { next: '?cursor=more-spaces' }, + }) + ) + await expect( + confluenceConnector.validateConfig( + 'token', + { domain: DOMAIN, spaceKey }, + { cloudId: CLOUD_ID, credentialDomain: DOMAIN } + ) + ).resolves.toEqual({ valid: true }) + expect(fetchMock).toHaveBeenCalledOnce() + const [input, options] = fetchMock.mock.calls[0] + const url = new URL(String(input)) + expect(url.pathname).toBe(`/ex/confluence/${CLOUD_ID}/wiki/api/v2/spaces`) + expect(url.searchParams.get('limit')).toBe('1') + expect(url.searchParams.has('keys')).toBe(false) + expect(options?.headers).toMatchObject({ Authorization: 'Bearer token' }) + } + ) + + it('does not let Confluence All bypass the service-account site binding', async () => { + await expect( + confluenceConnector.validateConfig( + 'token', + { domain: DOMAIN, spaceKey: ['*'] }, + { cloudId: CLOUD_ID, credentialDomain: 'other.atlassian.net' } + ) + ).resolves.toMatchObject({ valid: false, error: expect.stringContaining('must match') }) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('does not accept Confluence All when the provider denies space discovery', async () => { + fetchMock.mockResolvedValueOnce(Response.json({}, { status: 403 })) + await expect( + confluenceConnector.validateConfig( + 'token', + { domain: DOMAIN, spaceKey: '*' }, + { cloudId: CLOUD_ID } + ) + ).resolves.toMatchObject({ valid: false, error: expect.stringContaining('403') }) + }) + + it.each([{ value: '*' }, { value: ['*'] }])( + 'validates Jira All and its optional filter under the supplied account (%j)', + async ({ value: projectKey }) => { + fetchMock.mockImplementation(async () => Response.json({ issues: [] })) + await expect( + jiraConnector.validateConfig( + 'token', + { domain: DOMAIN, projectKey, jql: 'status = "Done"' }, + { cloudId: CLOUD_ID } + ) + ).resolves.toEqual({ valid: true }) + expect(fetchMock).toHaveBeenCalledTimes(2) + expect( + fetchMock.mock.calls.map(([input]) => new URL(String(input)).searchParams.get('jql')) + ).toEqual(['project IS NOT EMPTY', 'project IS NOT EMPTY AND (status = "Done")']) + for (const [input, options] of fetchMock.mock.calls) { + const url = new URL(String(input)) + expect(url.pathname).toBe(`/ex/jira/${CLOUD_ID}/rest/api/3/search/jql`) + expect(url.searchParams.get('maxResults')).toBe('1') + expect(options?.headers).toMatchObject({ Authorization: 'Bearer token' }) + } + } + ) + + it('does not let Jira All bypass current account site discovery', async () => { + fetchMock.mockResolvedValueOnce( + Response.json([{ id: 'other-cloud', url: 'https://other.atlassian.net' }]) + ) + await expect( + jiraConnector.validateConfig( + 'token', + { domain: DOMAIN, projectKey: ['*'] }, + { perMemberListing: true } + ) + ).resolves.toMatchObject({ + valid: false, + error: expect.stringContaining('Could not match Jira domain'), + }) + expect(fetchMock).toHaveBeenCalledOnce() + expect(String(fetchMock.mock.calls[0][0])).toBe( + 'https://api.atlassian.com/oauth/token/accessible-resources' + ) + }) + + it('does not accept Jira All when the provider denies issue search', async () => { + fetchMock.mockResolvedValueOnce(Response.json({}, { status: 403 })) + await expect( + jiraConnector.validateConfig( + 'token', + { domain: DOMAIN, projectKey: '*' }, + { cloudId: CLOUD_ID } + ) + ).resolves.toMatchObject({ valid: false, error: expect.stringContaining('403') }) + }) +}) diff --git a/apps/sim/connectors/confluence/attachments.test.ts b/apps/sim/connectors/confluence/attachments.test.ts new file mode 100644 index 00000000000..f74e033e96c --- /dev/null +++ b/apps/sim/connectors/confluence/attachments.test.ts @@ -0,0 +1,615 @@ +/** + * @vitest-environment node + */ +import JSZip from 'jszip' +import { PDFDocument, StandardFonts } from 'pdf-lib' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { DEFAULT_MAX_ERROR_BODY_BYTES, PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { parseBuffer } from '@/lib/file-parsers' +import { listConfluenceAttachments } from '@/connectors/confluence/attachments' +import { confluenceConnector } from '@/connectors/confluence/confluence' +import type { ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { CONNECTOR_MAX_FILE_BYTES } from '@/connectors/utils' + +const { secureDownload } = vi.hoisted(() => ({ secureDownload: vi.fn() })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithRetry: secureDownload, +})) + +const CONFIG = { domain: 'example.atlassian.net', spaceKey: 'ENG' } +const CONTEXT = { cloudId: 'cloud', spaceId: '1' } +const INPUT = { accessToken: 'token', cloudId: 'cloud', domain: CONFIG.domain } +const fetchMock = vi.fn() + +function file(overrides: Record = {}) { + return { + id: 'att123', + title: 'Guide.pdf', + status: 'current', + pageId: 'p1', + fileSize: 12, + version: { number: 2, createdAt: '2026-09-01T00:00:00Z' }, + webuiLink: '/spaces/ENG/pages/p1?preview=att123', + ...overrides, + } +} + +function parent(id = 'p1', type = 'page'): ExternalDocument { + return { + externalId: id, + title: id, + content: '', + mimeType: 'text/plain', + contentHash: id, + metadata: { contentType: type }, + } +} + +function fixture(attachment = file()) { + fetchMock.mockImplementation(async (input) => { + const url = new URL(String(input)) + const path = url.pathname + if (path.endsWith('/attachments/att123')) return Response.json(attachment) + if (path.endsWith('/pages/p1/attachments')) return Response.json({ results: [attachment] }) + if (path.endsWith('/spaces/1/pages')) + return Response.json({ + results: [ + { id: 'p1', title: 'Parent', status: 'current', spaceId: '1', version: { number: 1 } }, + ], + }) + if (path.endsWith('/pages/p1')) + return Response.json({ id: 'p1', status: 'current', spaceId: '1' }) + if (path.endsWith('/spaces/1')) return Response.json({ key: 'ENG' }) + if (path.endsWith('/pages/p1/labels')) + return Response.json({ results: [{ name: 'published' }] }) + if (path.endsWith('/download')) + return new Response(null, { + status: 302, + headers: { location: 'https://files.atlassian.net/signed-file?token=secret' }, + }) + if (path.endsWith('/spaces/1/permissions')) + return Response.json({ + results: [ + { + principal: { type: 'user', id: 'reader' }, + operation: { key: 'read', targetType: 'space' }, + }, + ], + }) + if (path.endsWith('/restriction/byOperation/read')) + return Response.json({ + restrictions: { user: { results: [] }, group: { results: [{ id: 'parent-readers' }] } }, + }) + if (path.endsWith('/ancestors')) return Response.json({ results: [] }) + throw new Error(`Unexpected request: ${url}`) + }) +} + +async function get(externalId = 'attachment:page:p1:att123', config = CONFIG) { + return confluenceConnector.getDocument('token', config, externalId, { ...CONTEXT }) +} + +beforeEach(() => { + fetchMock.mockReset() + secureDownload.mockReset().mockResolvedValue(new Response('binary bytes')) + vi.stubGlobal('fetch', fetchMock) +}) +afterEach(() => vi.unstubAllGlobals()) + +describe('Confluence attachment listing', () => { + it('lists PDF, DOC and DOCX stubs without downloading, and excludes unsupported or archived files', async () => { + fetchMock.mockResolvedValue( + Response.json({ + results: [ + file(), + file({ id: '2', title: 'Legacy.DOC' }), + file({ id: '3', title: 'Modern.docx' }), + file({ id: '4', title: 'image.png' }), + file({ id: '5', title: 'old.pdf', status: 'archived' }), + ], + }) + ) + const result = await listConfluenceAttachments({ + ...INPUT, + listParents: async () => ({ documents: [parent()], hasMore: false }), + }) + expect(result.documents.map((doc) => doc.externalId)).toEqual([ + 'p1', + 'attachment:page:p1:att123', + 'attachment:page:p1:2', + 'attachment:page:p1:3', + ]) + expect( + result.documents.slice(1).every((doc) => doc.contentDeferred && doc.content === '') + ).toBe(true) + expect(result.hasMore).toBe(false) + expect(secureDownload).not.toHaveBeenCalled() + expect(new URL(String(fetchMock.mock.calls[0][0])).searchParams.get('status')).toBe('current') + }) + + it('resumes a bounded parent queue with a fresh runtime context', async () => { + fetchMock.mockImplementation(async () => Response.json({ results: [] })) + const listParents = vi.fn( + async (): Promise => ({ + documents: Array.from({ length: 8 }, (_, index) => parent(`p${index}`)), + hasMore: false, + }) + ) + const first = await listConfluenceAttachments({ ...INPUT, listParents }) + expect(first.documents).toHaveLength(8) + expect(fetchMock).toHaveBeenCalledTimes(5) + expect(first.hasMore).toBe(true) + const last = await listConfluenceAttachments({ + ...INPUT, + listParents, + cursor: first.nextCursor, + syncContext: { totalDocsFetched: 8000 }, + }) + expect(last.hasMore).toBe(false) + expect(fetchMock).toHaveBeenCalledTimes(8) + expect(listParents).toHaveBeenCalledTimes(1) + }) + + it('keeps parent caps independent of attachment counts across worker resumes', async () => { + fetchMock.mockImplementation(async () => + Response.json({ + results: Array.from({ length: 10 }, (_, index) => file({ id: String(index) })), + }) + ) + const listParents = vi.fn( + async ( + cursor: string | undefined, + context: Record + ): Promise => { + expect(context.totalDocsFetched).toBe(cursor ? 1 : 0) + return { + documents: [parent()], + hasMore: !cursor, + nextCursor: cursor ? undefined : 'next-parent', + } + } + ) + const first = await listConfluenceAttachments({ ...INPUT, listParents }) + expect(first.documents).toHaveLength(11) + const last = await listConfluenceAttachments({ + ...INPUT, + listParents, + cursor: first.nextCursor, + syncContext: { totalDocsFetched: 11 }, + }) + expect(last.hasMore).toBe(false) + expect(listParents).toHaveBeenCalledTimes(2) + }) + + it('follows attachment continuations beyond the per-call request budget', async () => { + fetchMock.mockImplementation(async (input) => { + const cursor = Number(new URL(String(input)).searchParams.get('cursor') || 0) + return Response.json({ + results: [file({ id: String(cursor) })], + _links: cursor < 6 ? { next: `?cursor=${cursor + 1}` } : {}, + }) + }) + const listParents = vi.fn( + async (): Promise => ({ documents: [parent()], hasMore: false }) + ) + const first = await listConfluenceAttachments({ ...INPUT, listParents }) + const last = await listConfluenceAttachments({ + ...INPUT, + listParents, + cursor: first.nextCursor, + }) + expect([...first.documents, ...last.documents].map((doc) => doc.externalId)).toEqual([ + 'p1', + ...Array.from({ length: 7 }, (_, i) => `attachment:page:p1:${i}`), + ]) + expect(last.hasMore).toBe(false) + }) + + it.each([403, 401])( + 'preserves parent pages and cumulative partial progress for attachment access failure %s', + async (status) => { + fetchMock.mockImplementation(async () => + status === 401 + ? Response.json({ code: 401, message: 'Unauthorized; scope does not match' }, { status }) + : new Response(null, { status }) + ) + const listParents = vi.fn( + async (cursor: string | undefined): Promise => ({ + documents: [parent(cursor ? 'p2' : 'p1')], + hasMore: !cursor, + nextCursor: cursor ? undefined : 'next', + }) + ) + const first = await listConfluenceAttachments({ ...INPUT, listParents }) + const context: Record = {} + const last = await listConfluenceAttachments({ + ...INPUT, + listParents, + cursor: first.nextCursor, + syncContext: context, + }) + expect(first.documents.map((doc) => doc.externalId)).toEqual(['p1']) + expect(last.documents.map((doc) => doc.externalId)).toEqual(['p2']) + expect(last.listingFailures).toEqual({ + count: 2, + samples: ['p1', 'p2'].map((scope) => ({ + scope, + operation: 'confluence.attachments.list', + status, + reasons: [status === 401 ? 'attachment_scope_mismatch' : 'attachment_access_denied'], + })), + }) + expect(last.reconciliationSafe).toBe(false) + expect(context.reconciliationUnsafe).toBe(true) + } + ) + + it('continues accepting the parent cursor of a listing interrupted before attachments shipped', async () => { + const listParents = vi.fn( + async (): Promise => ({ documents: [], hasMore: false }) + ) + await listConfluenceAttachments({ + ...INPUT, + listParents, + cursor: 'space-batches:{"batch":1,"cursor":"provider"}', + syncContext: { totalDocsFetched: 27 }, + }) + expect(listParents).toHaveBeenCalledWith( + 'space-batches:{"batch":1,"cursor":"provider"}', + expect.objectContaining({ totalDocsFetched: 27 }) + ) + }) + + it('lists attachments even when only their version changed after the parent page watermark', async () => { + fixture() + const result = await confluenceConnector.listDocuments( + 'token', + CONFIG, + undefined, + { ...CONTEXT }, + new Date('2026-09-15') + ) + expect(result.documents.some((doc) => doc.externalId === 'attachment:page:p1:att123')).toBe( + true + ) + expect(fetchMock.mock.calls.some(([url]) => String(url).includes('lastModified'))).toBe(false) + }) + + it('surfaces known oversized files as skipped without downloading', async () => { + fetchMock.mockResolvedValue( + Response.json({ results: [file({ fileSize: CONNECTOR_MAX_FILE_BYTES + 1 })] }) + ) + const result = await listConfluenceAttachments({ + ...INPUT, + listParents: async () => ({ documents: [parent()], hasMore: false }), + }) + expect(result.documents[1].skippedReason).toContain('size limit') + expect(result.documents[1].contentDeferred).toBe(false) + }) + + it.each([ + { results: [], _links: { next: '?wrong=cursor' } }, + { results: [file({ pageId: 'another-parent' })] }, + { results: 'invalid' }, + ])('refuses an incomplete or mismatched attachment list %#', async (body) => { + fetchMock.mockResolvedValue(Response.json(body)) + await expect( + listConfluenceAttachments({ + ...INPUT, + listParents: async () => ({ documents: [parent()], hasMore: false }), + }) + ).rejects.toThrow() + }) + + it.each([ + null, + '{"code":401,"message":"Token is invalid","secret":"must-not-appear"}', + '{"code":403,"message":"Unauthorized; scope does not match"}', + 'invalid JSON', + 'x'.repeat(DEFAULT_MAX_ERROR_BODY_BYTES + 1), + ])('keeps unrecognized 401 responses as credential failures %#', async (body) => { + fetchMock.mockResolvedValue(new Response(body, { status: 401 })) + const error = await listConfluenceAttachments({ + ...INPUT, + listParents: async () => ({ documents: [parent()], hasMore: false }), + }).catch((error: unknown) => error) + expect(error).toMatchObject({ status: 401 }) + expect(confluenceConnector.isCredentialInvalidError?.(error)).toBe(true) + expect(String(error)).not.toContain('must-not-appear') + }) +}) + +describe('Confluence attachment hydration', () => { + it.each(['/attachments/att123', '/download'])( + 'reports missing attachment scope without invalidating the credential at %s', + async (endpoint) => { + fixture() + const healthy = fetchMock.getMockImplementation()! + fetchMock.mockImplementation(async (input, init) => + new URL(String(input)).pathname.endsWith(endpoint) + ? Response.json( + { + code: 401, + message: 'Unauthorized; scope does not match', + secret: 'must-not-appear', + }, + { status: 401 } + ) + : healthy(input, init) + ) + const error = await get().catch((error: unknown) => error) + expect(error).toBeInstanceOf(Error) + expect(String(error)).toContain('read:attachment:confluence') + expect(String(error)).not.toContain('must-not-appear') + expect(confluenceConnector.isCredentialInvalidError?.(error)).toBe(false) + expect(secureDownload).not.toHaveBeenCalled() + } + ) + + it.each(['/attachments/att123', '/download'])( + 'preserves expired-credential errors during hydration at %s', + async (endpoint) => { + fixture() + const healthy = fetchMock.getMockImplementation()! + fetchMock.mockImplementation(async (input, init) => + new URL(String(input)).pathname.endsWith(endpoint) + ? new Response(null, { status: 401 }) + : healthy(input, init) + ) + const error = await get().catch((error: unknown) => error) + expect(error).toMatchObject({ status: 401 }) + expect(confluenceConnector.isCredentialInvalidError?.(error)).toBe(true) + expect(secureDownload).not.toHaveBeenCalled() + } + ) + + it.each(['pdf', 'doc', 'docx'])( + 'hands an original %s file to the shared parser pipeline', + async (extension) => { + fixture(file({ title: `Guide.${extension}` })) + const listing = await confluenceConnector.listDocuments('token', CONFIG, undefined, { + ...CONTEXT, + }) + const doc = await get() + expect(doc?.sourceFile?.bytes.toString()).toBe('binary bytes') + expect(doc?.sourceFile?.fileName).toBe(`Guide.${extension}`) + expect(doc?.contentHash).toBe(listing.documents[1].contentHash) + expect(doc?.contentDeferred).toBe(false) + expect(doc?.mimeType).toBe(doc?.sourceFile?.mimeType) + const download = fetchMock.mock.calls.find(([url]) => String(url).includes('/download'))! + expect(String(download[0])).toContain( + '/content/p1/child/attachment/att123/download?version=2' + ) + expect(download[1]).toMatchObject({ + redirect: 'manual', + headers: { Authorization: 'Bearer token' }, + }) + expect(secureDownload.mock.calls[0][1]).toMatchObject({ + profile: 'contentFetch', + maxResponseBytes: CONNECTOR_MAX_FILE_BYTES, + }) + expect(secureDownload.mock.calls[0][1].headers).toBeUndefined() + } + ) + + it.each([ + '/spaces/ENG/pages/p1?preview=att123', + '/wiki/spaces/ENG/pages/p1?preview=att123', + 'https://example.atlassian.net/wiki/spaces/ENG/pages/p1?preview=att123', + ])('normalizes provider web links %s', async (webuiLink) => { + fixture(file({ webuiLink })) + expect((await get())?.sourceUrl).toBe( + 'https://example.atlassian.net/wiki/spaces/ENG/pages/p1?preview=att123' + ) + }) + + it('refuses a file moved after listing even when the token can read the new parent', async () => { + fixture(file({ pageId: 'private-page' })) + expect(await get()).toBeNull() + expect(secureDownload).not.toHaveBeenCalled() + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('refuses a current parent moved out of the selected space', async () => { + fixture() + const original = fetchMock.getMockImplementation()! + fetchMock.mockImplementation(async (input, init) => + String(input).endsWith('/spaces/1') + ? Response.json({ key: 'PRIVATE' }) + : original(input, init) + ) + expect(await get()).toBeNull() + expect(secureDownload).not.toHaveBeenCalled() + }) + + it('checks labels beyond the embedded fifty-label page before downloading', async () => { + fixture() + const original = fetchMock.getMockImplementation()! + fetchMock.mockImplementation(async (input, init) => { + const url = new URL(String(input)) + if (url.pathname.endsWith('/labels')) + return Response.json( + url.searchParams.has('cursor') + ? { results: [{ name: 'published' }] } + : { results: [{ name: 'other' }], _links: { next: '?cursor=second' } } + ) + return original(input, init) + }) + expect( + (await get(undefined, { ...CONFIG, labelFilter: 'published' } as typeof CONFIG))?.sourceFile + ).toBeDefined() + expect(fetchMock.mock.calls.filter(([url]) => String(url).includes('/labels'))).toHaveLength(2) + }) + + it('does not download when the current parent no longer matches a label filter', async () => { + fixture() + expect(await get(undefined, { ...CONFIG, labelFilter: 'missing' } as typeof CONFIG)).toBeNull() + expect(secureDownload).not.toHaveBeenCalled() + }) + + it('rejects non-HTTPS redirects before any download and guards subsequent hops', async () => { + fixture() + const original = fetchMock.getMockImplementation()! + fetchMock.mockImplementation(async (input, init) => + String(input).includes('/download') + ? new Response(null, { status: 302, headers: { location: 'http://127.0.0.1/secret' } }) + : original(input, init) + ) + await expect(get()).rejects.toThrow('unsafe') + expect(secureDownload).not.toHaveBeenCalled() + fixture() + await get() + expect(() => + secureDownload.mock.calls[0][1].assertRedirectTarget('http://internal/file') + ).toThrow('unsafe') + }) + + it('surfaces an oversized streamed download as a visible skip', async () => { + fixture() + secureDownload.mockRejectedValue( + new PayloadSizeLimitError({ label: 'download', maxBytes: CONNECTOR_MAX_FILE_BYTES }) + ) + expect((await get())?.skippedReason).toContain('size limit') + }) + + it('releases a rejected signed download response', async () => { + fixture() + const cancel = vi.fn() + secureDownload.mockResolvedValue(new Response(new ReadableStream({ cancel }), { status: 403 })) + await expect(get()).rejects.toThrow('Failed to download Confluence attachment: 403') + expect(cancel).toHaveBeenCalledOnce() + }) + + it('passes cancellation through metadata and signed download requests', async () => { + fixture() + const signal = new AbortController().signal + await confluenceConnector.getDocument('token', CONFIG, 'attachment:page:p1:att123', { + ...CONTEXT, + signal, + }) + expect(fetchMock.mock.calls.every(([, init]) => init?.signal instanceof AbortSignal)).toBe(true) + expect(secureDownload.mock.calls[0][1].signal).toBe(signal) + }) + + it.each(['pdf', 'docx'] as const)( + 'roundtrips genuine %s bytes through the public parser', + async (extension) => { + let bytes: Buffer + if (extension === 'pdf') { + const document = await PDFDocument.create() + const font = await document.embedFont(StandardFonts.Helvetica) + document.addPage().drawText('Confluence attachment text', { font, size: 14 }) + bytes = Buffer.from(await document.save()) + } else { + const zip = new JSZip() + zip.file( + '[Content_Types].xml', + '' + ) + zip.file( + '_rels/.rels', + '' + ) + zip.file( + 'word/document.xml', + 'Confluence attachment text' + ) + bytes = await zip.generateAsync({ type: 'nodebuffer' }) + } + fixture(file({ title: `Guide.${extension}`, fileSize: bytes.length })) + secureDownload.mockResolvedValue(new Response(bytes)) + const doc = await get() + expect(doc?.sourceFile).toBeDefined() + const parsed = await parseBuffer(doc!.sourceFile!.bytes, extension) + expect(parsed.content).toContain('Confluence attachment text') + } + ) +}) + +describe('Confluence attachment ACLs', () => { + it('hydrates blog post attachments and checks blog restrictions without page ancestors', async () => { + fixture(file({ pageId: undefined, blogPostId: 'b1' })) + const original = fetchMock.getMockImplementation()! + fetchMock.mockImplementation(async (input, init) => { + const path = new URL(String(input)).pathname + if (path.endsWith('/blogposts/b1/attachments')) + return Response.json({ results: [file({ pageId: undefined, blogPostId: 'b1' })] }) + if (path.endsWith('/blogposts/b1')) + return Response.json({ id: 'b1', spaceId: '1', status: 'current' }) + return original(input, init) + }) + const listing = await listConfluenceAttachments({ + ...INPUT, + listParents: async () => ({ documents: [parent('b1', 'blogpost')], hasMore: false }), + }) + const config = { ...CONFIG, contentType: 'blogpost' } + const doc = listing.documents[1] + const hydrated = await confluenceConnector.getDocument('token', config, doc.externalId, { + ...CONTEXT, + }) + expect(hydrated?.sourceFile).toBeDefined() + const acls = await confluenceConnector.getDocumentAcls!( + 'token', + config, + [doc], + { ...CONTEXT }, + { persistGroupMembership: vi.fn() } + ) + expect(acls[doc.externalId]).toEqual({ + acl: ['g:confluence:cloud:space-readers:1'], + requirements: [['g:confluence:cloud:parent-readers']], + }) + expect( + fetchMock.mock.calls.some(([url]) => String(url).includes('/content/b1/restriction')) + ).toBe(true) + expect(fetchMock.mock.calls.some(([url]) => String(url).includes('/ancestors'))).toBe(false) + }) + + it('uses the canonical parent restriction chain and persists its space audience', async () => { + fixture() + const listed = await confluenceConnector.listDocuments('token', CONFIG, undefined, { + ...CONTEXT, + }) + const persistGroupMembership = vi.fn().mockResolvedValue(undefined) + const acls = await confluenceConnector.getDocumentAcls!( + 'token', + CONFIG, + [listed.documents[1]], + { ...CONTEXT }, + { persistGroupMembership } + ) + expect(acls['attachment:page:p1:att123']).toEqual({ + acl: ['g:confluence:cloud:space-readers:1'], + requirements: [['g:confluence:cloud:parent-readers']], + }) + expect(persistGroupMembership).toHaveBeenCalledOnce() + expect( + fetchMock.mock.calls.some(([url]) => String(url).includes('/content/att123/restriction')) + ).toBe(false) + expect( + fetchMock.mock.calls.some(([url]) => String(url).includes('/pages/att123/ancestors')) + ).toBe(false) + }) + + it('fails closed for a moved attachment and for missing audience persistence', async () => { + fixture() + const listed = await confluenceConnector.listDocuments('token', CONFIG, undefined, { + ...CONTEXT, + }) + const doc = listed.documents[1] + expect( + await confluenceConnector.getDocumentAcls!('token', CONFIG, [doc], { ...CONTEXT }) + ).toEqual({}) + fixture(file({ pageId: 'new-parent' })) + expect( + await confluenceConnector.getDocumentAcls!( + 'token', + CONFIG, + [doc], + { ...CONTEXT }, + { persistGroupMembership: vi.fn() } + ) + ).toEqual({}) + }) +}) diff --git a/apps/sim/connectors/confluence/attachments.ts b/apps/sim/connectors/confluence/attachments.ts new file mode 100644 index 00000000000..087e84faa20 --- /dev/null +++ b/apps/sim/connectors/confluence/attachments.ts @@ -0,0 +1,478 @@ +import { z } from 'zod' +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { fetchWithRetry, secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { + createRetryableHttpError, + readBoundedHttpErrorPayload, +} from '@/lib/knowledge/documents/utils' +import { extractCursor } from '@/connectors/confluence/cursor' +import { listingFailuresSchema, MAX_LISTING_FAILURE_SAMPLES } from '@/connectors/listing-failures' +import { isAllSourceItems } from '@/connectors/selection' +import type { ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { + CONNECTOR_MAX_FILE_BYTES, + connectorFileExtension, + markSkipped, + parseMultiValue, + pipelineParsedMimeType, + readBodyWithLimit, + sizeLimitSkipReason, + stubOrSkipBySize, +} from '@/connectors/utils' + +const ATTACHMENT_PREFIX = 'attachment:' +const CURSOR_PREFIX = 'attachments:' +const PAGE_SIZE = 50 +const REQUESTS_PER_CALL = 5 +const MAX_CURSOR_BYTES = 512 * 1024 +const MAX_METADATA_BYTES = 2 * 1024 * 1024 +const FILE_EXTENSIONS = new Set(['pdf', 'doc', 'docx']) +const boundedId = z.string().min(1).max(254) +const providerCursor = z.string().min(1).max(8192) +const parentSchema = z.object({ id: boundedId, type: z.enum(['page', 'blogpost']) }) +const cursorSchema = z.object({ + parentCursor: z + .string() + .min(1) + .max(32 * 1024) + .optional(), + parents: z.array(parentSchema).max(500), + attachmentCursor: providerCursor.optional(), + parentsListed: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + parentsDone: z.boolean(), + failures: listingFailuresSchema.optional(), +}) +const attachmentSchema = z.object({ + id: boundedId, + title: z.string().min(1).max(4096), + status: z.string(), + pageId: boundedId.optional(), + blogPostId: boundedId.optional(), + customContentId: boundedId.optional(), + fileSize: z.number().int().nonnegative().optional(), + version: z.object({ number: z.number().int().positive(), createdAt: z.string().optional() }), + webuiLink: z.string().optional(), + _links: z.object({ webui: z.string().optional() }).optional(), +}) +const attachmentPageSchema = z.object({ + results: z.array(attachmentSchema).max(250), + _links: z.object({ next: providerCursor.optional() }).optional(), +}) +const scopeMismatchSchema = z.object({ + code: z.literal(401), + message: z.literal('Unauthorized; scope does not match'), +}) + +type Attachment = z.infer +type AttachmentCursor = z.infer +type AttachmentParent = z.infer + +interface AttachmentRequest { + accessToken: string + cloudId: string + domain: string + syncContext?: Record +} + +class ConfluenceAttachmentScopeError extends Error { + constructor() { + super('Confluence attachments require the read:attachment:confluence credential scope.') + this.name = 'ConfluenceAttachmentScopeError' + } +} + +/** Atlassian reports missing endpoint scopes as 401 without invalidating the token itself. */ +async function unauthorizedAttachmentError(response: Response): Promise { + const payload = await readBoundedHttpErrorPayload(response) + if (payload.ok) { + try { + if (scopeMismatchSchema.safeParse(JSON.parse(payload.body)).success) { + return new ConfluenceAttachmentScopeError() + } + } catch { + /** An unrecognized response remains an authentication failure. */ + } + } + return createRetryableHttpError({ status: response.status, headers: response.headers }) +} + +function signalFor(input: AttachmentRequest): AbortSignal | undefined { + const signal = input.syncContext?.signal + return signal instanceof AbortSignal ? signal : undefined +} + +function apiBase(cloudId: string): string { + return `https://api.atlassian.com/ex/confluence/${encodeURIComponent(cloudId)}/wiki` +} + +async function requestMetadata(input: AttachmentRequest, path: string): Promise { + return fetchWithRetry(`${apiBase(input.cloudId)}${path}`, { + headers: { Authorization: `Bearer ${input.accessToken}`, Accept: 'application/json' }, + signal: signalFor(input), + redirect: 'error', + }) +} + +async function readMetadata(response: Response): Promise { + if (!response.ok) { + if (response.status === 401) throw await unauthorizedAttachmentError(response) + await response.body?.cancel() + throw new Error( + response.status === 403 + ? 'Confluence attachment access was denied. Check the parent content permissions and the read:attachment:confluence credential scope.' + : `Failed to read Confluence attachment metadata: ${response.status}` + ) + } + return readResponseJsonWithLimit(response, { + maxBytes: MAX_METADATA_BYTES, + label: 'Confluence attachment metadata', + }) +} + +function attachmentParent(attachment: Attachment): AttachmentParent { + if (attachment.customContentId || Boolean(attachment.pageId) === Boolean(attachment.blogPostId)) { + throw new Error('Confluence attachment has no unambiguous page or blog post parent') + } + return attachment.pageId + ? { id: attachment.pageId, type: 'page' } + : { id: attachment.blogPostId!, type: 'blogpost' } +} + +function sameParent(left: AttachmentParent, right: AttachmentParent): boolean { + return left.id === right.id && left.type === right.type +} + +function attachmentMimeType(attachment: Attachment): string | undefined { + const extension = connectorFileExtension(attachment.title) + return extension && FILE_EXTENSIONS.has(extension) + ? pipelineParsedMimeType(attachment.title) + : undefined +} + +function attachmentToStub( + attachment: Attachment, + parent: AttachmentParent, + domain: string +): ExternalDocument { + const fallback = `https://${domain}/wiki/pages/viewpage.action?pageId=${encodeURIComponent(parent.id)}` + const link = attachment.webuiLink ?? attachment._links?.webui + let sourceUrl = fallback + if (link) { + try { + const relative = link.startsWith('/') && !link.startsWith('/wiki/') ? `/wiki${link}` : link + const url = new URL(relative, `https://${domain}/wiki/`) + if (url.origin === `https://${domain}`) sourceUrl = url.toString() + } catch { + sourceUrl = fallback + } + } + return { + externalId: `${ATTACHMENT_PREFIX}${parent.type}:${encodeURIComponent(parent.id)}:${encodeURIComponent(attachment.id)}`, + title: attachment.title, + content: '', + contentDeferred: true, + mimeType: attachmentMimeType(attachment) ?? 'application/octet-stream', + contentHash: `confluence:attachment-v1:${attachment.id}:${parent.type}:${parent.id}:${attachment.version.number}`, + sourceUrl, + metadata: { + contentType: 'attachment', + parentId: parent.id, + parentContentType: parent.type, + version: attachment.version.number, + lastModified: attachment.version.createdAt, + fileSize: attachment.fileSize, + }, + } +} + +function decodeCursor(cursor: string | undefined, totalFetched: unknown): AttachmentCursor { + if (!cursor?.startsWith(CURSOR_PREFIX)) { + return { + parents: [], + parentCursor: cursor, + parentsDone: false, + parentsListed: typeof totalFetched === 'number' ? totalFetched : 0, + } + } + if (Buffer.byteLength(cursor, 'utf8') > MAX_CURSOR_BYTES) { + throw new Error('Confluence attachment continuation exceeds its size limit') + } + const parsed = cursorSchema.safeParse(JSON.parse(cursor.slice(CURSOR_PREFIX.length))) + if (!parsed.success || (parsed.data.attachmentCursor && parsed.data.parents.length === 0)) { + throw new Error('Invalid Confluence attachment continuation. Restart the sync.') + } + return parsed.data +} + +/** + * Visits attachment metadata beneath the already-filtered parents. The cursor + * contains only a bounded parent queue, never bodies or binary data. Parent caps + * remain independent of the sync engine's combined page-and-attachment count. + */ +export async function listConfluenceAttachments( + input: AttachmentRequest & { + cursor?: string + listParents: ( + cursor: string | undefined, + syncContext: Record + ) => Promise + } +): Promise { + const state = decodeCursor(input.cursor, input.syncContext?.totalDocsFetched) + const documents: ExternalDocument[] = [] + if (state.parents.length === 0 && !state.parentsDone) { + const parentContext = { ...input.syncContext, totalDocsFetched: state.parentsListed } + const page = await input.listParents(state.parentCursor, parentContext) + if (page.documents.length > 500 || (page.hasMore && !page.nextCursor)) { + throw new Error('Confluence returned an invalid parent listing') + } + if (input.syncContext) Object.assign(input.syncContext, parentContext) + state.parentsListed += page.documents.length + state.parentCursor = page.nextCursor + state.parentsDone = !page.hasMore + state.parents = page.documents.map((doc) => + parentSchema.parse({ id: doc.externalId, type: doc.metadata?.contentType ?? 'page' }) + ) + documents.push(...page.documents) + } + + for (let request = 0; state.parents.length > 0 && request < REQUESTS_PER_CALL; request++) { + const parent = state.parents[0] + const query = new URLSearchParams({ limit: String(PAGE_SIZE), status: 'current' }) + if (state.attachmentCursor) query.set('cursor', state.attachmentCursor) + const response = await requestMetadata( + input, + `/api/v2/${parent.type}s/${encodeURIComponent(parent.id)}/attachments?${query}` + ) + let page: z.infer + try { + page = attachmentPageSchema.parse(await readMetadata(response)) + } catch (error) { + const missingScope = error instanceof ConfluenceAttachmentScopeError + if (!missingScope && response.status !== 403 && response.status !== 404) throw error + state.failures ??= { count: 0, samples: [] } + state.failures.count += 1 + if (state.failures.samples.length < MAX_LISTING_FAILURE_SAMPLES) { + state.failures.samples.push({ + scope: parent.id, + operation: 'confluence.attachments.list', + status: response.status, + reasons: [missingScope ? 'attachment_scope_mismatch' : 'attachment_access_denied'], + }) + } + state.parents.shift() + state.attachmentCursor = undefined + continue + } + for (const attachment of page.results) { + if (attachment.status !== 'current' || !attachmentMimeType(attachment)) continue + if (!sameParent(attachmentParent(attachment), parent)) { + throw new Error('Confluence returned an attachment belonging to another parent') + } + documents.push( + stubOrSkipBySize( + attachmentToStub(attachment, parent, input.domain), + attachment.fileSize, + CONNECTOR_MAX_FILE_BYTES + ) + ) + } + const next = page._links?.next + const nextCursor = extractCursor(next) + if (next && (!nextCursor || nextCursor === state.attachmentCursor)) { + throw new Error('Confluence returned an invalid or repeated attachment continuation') + } + state.attachmentCursor = nextCursor + if (!nextCursor) state.parents.shift() + } + if (state.failures && input.syncContext) input.syncContext.reconciliationUnsafe = true + const hasMore = state.parents.length > 0 || !state.parentsDone + const nextCursor = hasMore ? CURSOR_PREFIX + JSON.stringify(state) : undefined + if (nextCursor && Buffer.byteLength(nextCursor, 'utf8') > MAX_CURSOR_BYTES) { + throw new Error('Confluence attachment continuation exceeds its size limit') + } + return { + documents, + hasMore, + nextCursor, + ...(state.failures ? { listingFailures: state.failures, reconciliationSafe: false } : {}), + } +} + +export function isConfluenceAttachment(externalId: string): boolean { + return externalId.startsWith(ATTACHMENT_PREFIX) +} + +async function readAttachment( + input: AttachmentRequest, + externalId: string +): Promise { + const [type, parentId, attachmentId, extra] = externalId + .slice(ATTACHMENT_PREFIX.length) + .split(':') + if (extra !== undefined || !parentId || !attachmentId) + throw new Error('Invalid Confluence attachment identity') + const expectedParent = parentSchema.parse({ type, id: decodeURIComponent(parentId) }) + const id = boundedId.parse(decodeURIComponent(attachmentId)) + const response = await requestMetadata(input, `/api/v2/attachments/${encodeURIComponent(id)}`) + if (response.status === 404) { + await response.body?.cancel() + return null + } + const attachment = attachmentSchema.parse(await readMetadata(response)) + if (attachment.id !== id) throw new Error('Confluence returned another attachment') + if (attachment.status !== 'current' || attachment.customContentId) return null + return sameParent(attachmentParent(attachment), expectedParent) ? attachment : null +} + +/** Verifies the current parent's configured scope rather than trusting stored metadata. */ +async function parentLocation( + input: AttachmentRequest, + parent: AttachmentParent, + sourceConfig: Record +): Promise<{ spaceId: string; contentType: 'page' | 'blogpost' } | null> { + const selectedType = sourceConfig.contentType || 'page' + if (selectedType !== 'all' && selectedType !== parent.type) return null + const path = `/api/v2/${parent.type}s/${encodeURIComponent(parent.id)}` + const response = await requestMetadata(input, path) + if (response.status === 404) { + await response.body?.cancel() + return null + } + const page = z + .object({ id: boundedId, status: z.string(), spaceId: boundedId }) + .parse(await readMetadata(response)) + if (page.id !== parent.id || page.status !== 'current') return null + if (!isAllSourceItems(sourceConfig.spaceKey)) { + const space = z + .object({ key: z.string() }) + .parse( + await readMetadata( + await requestMetadata(input, `/api/v2/spaces/${encodeURIComponent(page.spaceId)}`) + ) + ) + if (!parseMultiValue(sourceConfig.spaceKey).includes(space.key)) return null + } + const labels = parseMultiValue(sourceConfig.labelFilter) + if (labels.length > 0) { + const seen = new Set() + let cursor: string | undefined + let matched = false + for (let count = 0; count < 100; count++) { + const query = new URLSearchParams({ limit: '250' }) + if (cursor) query.set('cursor', cursor) + const result = z + .object({ + results: z.array(z.object({ name: z.string() })).max(250), + _links: z.object({ next: providerCursor.optional() }).optional(), + }) + .parse(await readMetadata(await requestMetadata(input, `${path}/labels?${query}`))) + if (result.results.some((label) => labels.includes(label.name))) { + matched = true + break + } + const next = result._links?.next + if (!next) break + cursor = extractCursor(next) + if (!cursor || seen.has(cursor) || count === 99) { + throw new Error('Confluence parent labels could not be completely verified') + } + seen.add(cursor) + } + if (!matched) return null + } + return { spaceId: page.spaceId, contentType: parent.type } +} + +/** Attachment ACLs authorize the freshly verified parent, never the attachment ID as a page. */ +export async function locateConfluenceAttachment( + input: AttachmentRequest, + sourceConfig: Record, + doc: ExternalDocument +): Promise<{ id: string; spaceId: string; contentType: 'page' | 'blogpost' } | null> { + const attachment = await readAttachment(input, doc.externalId) + if (!attachment) return null + const parent = attachmentParent(attachment) + if (doc.metadata?.parentId !== parent.id || doc.metadata?.parentContentType !== parent.type) + return null + const location = await parentLocation(input, parent, sourceConfig) + return location ? { id: parent.id, ...location } : null +} + +function assertDownloadUrl(value: string): void { + const url = new URL(value) + if (url.protocol !== 'https:' || url.username || url.password) { + throw new Error('Confluence returned an unsafe attachment download URL') + } +} + +/** Downloads a version-pinned original for the shared PDF/OCR and Word parsing pipeline. */ +export async function getConfluenceAttachment( + input: AttachmentRequest, + sourceConfig: Record, + externalId: string +): Promise { + const attachment = await readAttachment(input, externalId) + if (!attachment) return null + const parent = attachmentParent(attachment) + if (!(await parentLocation(input, parent, sourceConfig))) return null + const stub = attachmentToStub(attachment, parent, input.domain) + const mimeType = attachmentMimeType(attachment) + if (!mimeType) { + return { + ...markSkipped(stub, 'Attachment is no longer a PDF or Word document'), + skippedExistingDisposition: 'replace', + } + } + if (attachment.fileSize && attachment.fileSize > CONNECTOR_MAX_FILE_BYTES) { + return markSkipped(stub, sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES)) + } + const downloadUrl = `${apiBase(input.cloudId)}/rest/api/content/${encodeURIComponent(parent.id)}/child/attachment/${encodeURIComponent(attachment.id)}/download?version=${attachment.version.number}` + const redirect = await fetchWithRetry(downloadUrl, { + headers: { Authorization: `Bearer ${input.accessToken}` }, + redirect: 'manual', + signal: signalFor(input), + }) + if (redirect.status === 401) throw await unauthorizedAttachmentError(redirect) + const location = redirect.headers.get('location') + await redirect.body?.cancel() + if (redirect.status === 404) return null + if (redirect.status !== 302 || !location) { + throw new Error( + `Confluence attachment download did not return a download redirect: ${redirect.status}` + ) + } + const target = new URL(location, downloadUrl).toString() + assertDownloadUrl(target) + try { + const response = await secureFetchWithRetry(target, { + profile: 'contentFetch', + method: 'GET', + maxResponseBytes: CONNECTOR_MAX_FILE_BYTES, + signal: signalFor(input), + assertRedirectTarget: assertDownloadUrl, + stripAuthOnRedirect: true, + logUrlValidationDetails: false, + }) + if (!response.ok) { + await response.body?.cancel() + throw new Error(`Failed to download Confluence attachment: ${response.status}`) + } + const bytes = await readBodyWithLimit(response, CONNECTOR_MAX_FILE_BYTES) + if (!bytes) return markSkipped(stub, sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES)) + if (bytes.length === 0) + return { + ...markSkipped(stub, 'Document contains no extractable text'), + skippedExistingDisposition: 'replace', + } + return { + ...stub, + contentDeferred: false, + sourceFile: { bytes, fileName: attachment.title, mimeType }, + } + } catch (error) { + if (isPayloadSizeLimitError(error)) + return markSkipped(stub, sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES)) + throw error + } +} diff --git a/apps/sim/connectors/confluence/bulk-spaces.test.ts b/apps/sim/connectors/confluence/bulk-spaces.test.ts index f2e670030a6..0b1227efc20 100644 --- a/apps/sim/connectors/confluence/bulk-spaces.test.ts +++ b/apps/sim/connectors/confluence/bulk-spaces.test.ts @@ -14,7 +14,11 @@ function requestUrl(input: string | URL | Request): URL { beforeEach(() => { fetchMock.mockReset() - vi.stubGlobal('fetch', fetchMock) + vi.stubGlobal('fetch', (input: Parameters[0], init?: RequestInit) => { + return requestUrl(input).pathname.endsWith('/attachments') + ? Promise.resolve(Response.json({ results: [] })) + : fetchMock(input, init) + }) }) afterEach(() => vi.unstubAllGlobals()) diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index 894292d982a..b8428157b8d 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -7,7 +7,6 @@ import { AtlassianSiteNotMatchedError, } from '@/lib/atlassian/discovery' import { - buildLastModifiedClause, confluenceConnector, confluenceStorageToPlainText, confluenceViewToPlainText, @@ -19,6 +18,97 @@ import { } from '@/connectors/confluence/confluence' import { extractCursor } from '@/connectors/confluence/cursor' +/** Existing page fixtures have no files; attachment traversal has its own regression suite. */ +function stubFetchWithoutAttachments(mockFetch: typeof fetch): void { + vi.stubGlobal( + 'fetch', + vi.fn((input: Parameters[0], init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)) + return url.pathname.endsWith('/attachments') + ? Promise.resolve(Response.json({ results: [] })) + : mockFetch(input, init) + }) + ) +} + +describe('Confluence dynamic All scope', () => { + afterEach(() => vi.unstubAllGlobals()) + + it('lists newly accessible spaces on each sync, including old content, and follows pagination', async () => { + const fetchMock = vi.fn() + const page = (id: string, key: string) => ({ + id, + type: 'page', + status: 'current', + title: id, + space: { key }, + version: { number: 1 }, + }) + fetchMock + .mockResolvedValueOnce( + new Response( + JSON.stringify({ results: [page('1', 'ENG')], _links: { next: '?cursor=next' } }) + ) + ) + .mockResolvedValueOnce(new Response(JSON.stringify({ results: [page('2', 'HR')] }))) + .mockResolvedValueOnce(new Response(JSON.stringify({ results: [page('3', 'NEW')] }))) + stubFetchWithoutAttachments(fetchMock) + const config = { + domain: 'example.atlassian.net', + spaceKey: ['*'], + contentType: 'all', + labelFilter: 'published', + } + const context = { cloudId: 'cloud-1' } + const first = await confluenceConnector.listDocuments( + 'token', + config, + undefined, + context, + new Date() + ) + const second = await confluenceConnector.listDocuments( + 'token', + config, + first.nextCursor, + context, + new Date() + ) + const nextSync = await confluenceConnector.listDocuments( + 'token', + config, + undefined, + { cloudId: 'cloud-1' }, + new Date() + ) + expect(first.hasMore).toBe(true) + expect(second.documents[0].externalId).toBe('2') + expect(second.hasMore).toBe(false) + expect(nextSync.documents[0].externalId).toBe('3') + const urls = fetchMock.mock.calls.map(([input]) => new URL(String(input))) + expect(urls.map((url) => url.searchParams.get('cql'))).toEqual( + Array(3).fill('type in ("page","blogpost") AND label="published"') + ) + expect(urls[1].searchParams.get('cursor')).toBe('next') + }) + + it.each([ + {}, + { results: [], _links: { next: '?broken=cursor' } }, + { results: [], _links: { next: '?cursor=repeat' } }, + ])('rejects an incomplete search response %#', async (body) => { + stubFetchWithoutAttachments(vi.fn().mockResolvedValue(new Response(JSON.stringify(body)))) + await expect( + confluenceConnector.listDocuments( + 'token', + { domain: 'example.atlassian.net', spaceKey: '*' }, + 'repeat', + { cloudId: 'cloud-1' } + ) + ).rejects.toThrow(/invalid|repeated/) + }) +}) + describe('Confluence service-account scopes', () => { it('requests metadata and role reads needed for complete mirrored ACLs', () => { expect(confluenceConnector.auth.mode).toBe('oauth') @@ -60,33 +150,13 @@ describe('escapeCql', () => { }) }) -describe('buildLastModifiedClause', () => { - const now = new Date('2026-09-01T12:00:00Z') - - it.concurrent('rounds the watermark up to whole minutes relative to the server clock', () => { - expect(buildLastModifiedClause(new Date('2026-09-01T11:30:30Z'), now)).toBe( - 'lastModified >= now("-30m")' - ) - }) - - it.concurrent('never asks for less than a minute', () => { - expect(buildLastModifiedClause(now, now)).toBe('lastModified >= now("-1m")') - expect(buildLastModifiedClause(new Date(now.getTime() + 60_000), now)).toBe( - 'lastModified >= now("-1m")' - ) - }) -}) - describe('Confluence rejected credentials', () => { afterEach(() => vi.unstubAllGlobals()) it.each(['discovery', 'space', 'pages', 'cql', 'content'] as const)( 'preserves authenticated401 at the %s boundary', async (boundary) => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => new Response('', { status: 401 })) - ) + stubFetchWithoutAttachments(vi.fn(async () => new Response('', { status: 401 }))) const config = { domain: 'revocation-fixture.atlassian.net', spaceKey: 'ENG', @@ -548,69 +618,6 @@ describe('confluenceViewToPlainText', () => { }) }) -describe('confluence incremental CQL listing', () => { - const fetchMock = - vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise>() - - function jsonResponse(body: unknown): Response { - return new Response(JSON.stringify(body), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }) - } - - function cqlOfCall(index: number): string | null { - return new URL(String(fetchMock.mock.calls[index][0])).searchParams.get('cql') - } - - beforeEach(() => { - vi.useFakeTimers() - fetchMock.mockReset() - vi.stubGlobal('fetch', fetchMock) - }) - - afterEach(() => { - vi.unstubAllGlobals() - vi.useRealTimers() - }) - - it('keeps one lastModified clause across pages that straddle a minute boundary', async () => { - const lastSyncAt = new Date('2026-09-01T11:30:00Z') - const config = { domain: 'example.atlassian.net', spaceKey: 'ENG' } - const syncContext: Record = { cloudId: 'cloud-1' } - fetchMock - .mockResolvedValueOnce( - jsonResponse({ - results: [], - _links: { next: '/wiki/rest/api/content/search?cursor=page-2&cql=ignored' }, - }) - ) - .mockResolvedValueOnce(jsonResponse({ results: [] })) - - vi.setSystemTime(new Date('2026-09-01T12:00:59Z')) - const first = await confluenceConnector.listDocuments( - 'token', - config, - undefined, - syncContext, - lastSyncAt - ) - expect(first.nextCursor).toBe('page-2') - - vi.setSystemTime(new Date('2026-09-01T12:01:01Z')) - await confluenceConnector.listDocuments( - 'token', - config, - first.nextCursor, - syncContext, - lastSyncAt - ) - - expect(cqlOfCall(0)).toContain('lastModified >= now("-31m")') - expect(cqlOfCall(1)).toBe(cqlOfCall(0)) - }) -}) - describe('Confluence service-account site binding', () => { const config = { domain: 'other.atlassian.net', spaceKey: 'ENG' } const context = { cloudId: 'cloud-1', credentialDomain: 'bound.atlassian.net' } @@ -619,7 +626,7 @@ describe('Confluence service-account site binding', () => { beforeEach(() => { fetchMock.mockReset() fetchMock.mockRejectedValue(new Error('Unexpected provider request')) - vi.stubGlobal('fetch', fetchMock) + stubFetchWithoutAttachments(fetchMock) }) afterEach(() => vi.unstubAllGlobals()) @@ -696,7 +703,7 @@ describe('Confluence listing limits', () => { beforeEach(() => { fetchMock.mockReset() - vi.stubGlobal('fetch', fetchMock) + stubFetchWithoutAttachments(fetchMock) context = { cloudId: 'cloud-1', spaceId: 'space-1' } }) @@ -807,7 +814,9 @@ describe('Confluence listing limits', () => { ) expect(result.documents).toHaveLength(4) expect(result.hasMore).toBe(true) - expect(JSON.parse(result.nextCursor!)).toEqual({ + expect( + JSON.parse(JSON.parse(result.nextCursor!.slice('attachments:'.length)).parentCursor) + ).toEqual({ page: 'next-page', blog: 'next-blog', pagesDone: false, @@ -954,8 +963,7 @@ describe('Confluence permission-scoped content', () => { const view = '

Shared handbook

CONFIDENTIAL SALARY DATA

Local information

' beforeEach(() => { - vi.stubGlobal( - 'fetch', + stubFetchWithoutAttachments( vi.fn(async (input: string | URL | Request) => { const format = new URL(String(input)).searchParams.get('body-format') return new Response( @@ -1198,6 +1206,7 @@ describe('Confluence permission-scoped content', () => { } vi.mocked(fetch).mockImplementation(async (input) => { const url = new URL(String(input)) + if (url.pathname.endsWith('/attachments')) return Response.json({ results: [] }) if (url.pathname.endsWith('/spaces')) { return new Response(JSON.stringify({ results: [{ id: 'space-1', key: 'ENG' }] })) } @@ -1297,7 +1306,7 @@ describe('confluence mirrored permissions', () => { beforeEach(() => { fetchMock.mockReset() - vi.stubGlobal('fetch', fetchMock) + stubFetchWithoutAttachments(fetchMock) }) afterEach(() => { @@ -1316,12 +1325,13 @@ describe('confluence mirrored permissions', () => { 'token', { domain: 'example.atlassian.net', spaceKey: ['ENG', 'HR'] }, [page('eng-page', 'ENG'), page('hr-post', 'HR', 'blogpost')], - { cloudId: 'cloud-1' } + { cloudId: 'cloud-1' }, + { persistGroupMembership: vi.fn().mockResolvedValue(undefined) } ) expect(acls).toEqual({ - 'eng-page': ['s:confluence:-:acc-eng'], - 'hr-post': ['s:confluence:-:acc-hr'], + 'eng-page': ['g:confluence:cloud-1:space-readers:1'], + 'hr-post': ['g:confluence:cloud-1:space-readers:2'], }) /** A blog post has no ancestors and is never asked for them. */ const asked = fetchMock.mock.calls.map(([input]) => new URL(String(input)).pathname) @@ -1344,10 +1354,66 @@ describe('confluence mirrored permissions', () => { 'token', { domain: 'example.atlassian.net', spaceKey: 'ENG' }, [page('eng-page', 'ENG'), page('broken', 'ENG')], - { cloudId: 'cloud-1' } + { cloudId: 'cloud-1' }, + { persistGroupMembership: vi.fn().mockResolvedValue(undefined) } + ) + + expect(acls).toEqual({ 'eng-page': ['g:confluence:cloud-1:space-readers:1'] }) + }) + + it('refreshes and persists a shared space audience once for the entire document batch', async () => { + site() + const persistGroupMembership = vi.fn().mockResolvedValue(undefined) + const acls = await confluenceConnector.getDocumentAcls?.( + 'token', + { domain: 'example.atlassian.net', spaceKey: 'ENG' }, + [page('first', 'ENG'), page('second', 'ENG')], + { cloudId: 'cloud-1' }, + { persistGroupMembership } ) + expect(Object.keys(acls ?? {})).toEqual(['first', 'second']) + expect(persistGroupMembership).toHaveBeenCalledOnce() + expect(persistGroupMembership).toHaveBeenCalledWith({ + providerId: 'confluence', + tenantId: 'cloud-1', + group: expect.objectContaining({ id: 'space-readers:1' }), + memberTokens: ['s:confluence:-:acc-eng'], + }) + expect( + fetchMock.mock.calls.filter(([url]) => + String(url).endsWith('/spaces/1/permissions?limit=250') + ) + ).toHaveLength(1) + }) - expect(acls).toEqual({ 'eng-page': ['s:confluence:-:acc-eng'] }) + it('withholds only the failed space when its audience cannot be refreshed', async () => { + site() + const healthy = fetchMock.getMockImplementation()! + fetchMock.mockImplementation(async (input, init) => + String(input).includes('/spaces/2/permissions') ? jsonResponse({}, 403) : healthy(input, init) + ) + const persistGroupMembership = vi.fn().mockResolvedValue(undefined) + const acls = await confluenceConnector.getDocumentAcls?.( + 'token', + { domain: 'example.atlassian.net', spaceKey: ['ENG', 'HR'] }, + [page('eng', 'ENG'), page('hr', 'HR')], + { cloudId: 'cloud-1' }, + { persistGroupMembership } + ) + expect(acls).toEqual({ eng: ['g:confluence:cloud-1:space-readers:1'] }) + expect(persistGroupMembership).toHaveBeenCalledOnce() + }) + + it('withholds a space ACL when its verified audience could not be persisted', async () => { + site() + const acls = await confluenceConnector.getDocumentAcls?.( + 'token', + { domain: 'example.atlassian.net', spaceKey: 'ENG' }, + [page('eng', 'ENG')], + { cloudId: 'cloud-1' }, + { persistGroupMembership: vi.fn().mockRejectedValue(new Error('database unavailable')) } + ) + expect(acls).toEqual({}) }) it('loads restrictions above the first ancestor batch even when the page and parent already restrict access', async () => { @@ -1375,10 +1441,11 @@ describe('confluence mirrored permissions', () => { 'token', { domain: 'example.atlassian.net', spaceKey: 'ENG' }, [page('eng-page', 'ENG')], - { cloudId: 'cloud-1' } + { cloudId: 'cloud-1' }, + { persistGroupMembership: vi.fn().mockResolvedValue(undefined) } ) expect(acls?.['eng-page']).toEqual({ - acl: ['s:confluence:-:acc-eng'], + acl: ['g:confluence:cloud-1:space-readers:1'], requirements: expect.arrayContaining([ ['g:confluence:cloud-1:group-eng-page'], ['g:confluence:cloud-1:group-parent'], diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index f0da25bdb44..8a079f06fe6 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -18,17 +18,29 @@ import { type RetryOptions, VALIDATE_RETRY_OPTIONS, } from '@/lib/knowledge/documents/utils' +import { + getConfluenceAttachment, + isConfluenceAttachment, + listConfluenceAttachments, + locateConfluenceAttachment, +} from '@/connectors/confluence/attachments' import { extractCursor } from '@/connectors/confluence/cursor' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' import { describeContent, getReadRestriction, listAncestorIds, - listSpaceReadPrincipals, + listConfluenceSpaceMembership, openConfluenceDirectory, validateConfluencePermissionAccess, } from '@/connectors/confluence/permissions' -import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { getSourceSelectionError, isAllSourceItems } from '@/connectors/selection' +import type { + ConnectorAclContext, + ConnectorConfig, + ExternalDocument, + ExternalDocumentList, +} from '@/connectors/types' import { htmlToPlainText, joinTagArray, @@ -598,6 +610,7 @@ const ACL_CONCURRENCY = 8 /** Where a listed piece of content lives, as the permission pass needs it. */ interface ContentLocation { + id: string spaceId: string contentType: string } @@ -606,9 +619,9 @@ interface ContentLocation { * Resolves who may read each listed page. * * Confluence reports a page's restrictions only when asked for that page, so - * unlike Drive this cannot ride along with the listing. Two things are cached - * for the batch: each space's read principals and each page's restriction, - * which may be consulted by many descendants. + * unlike Drive this cannot ride along with the listing. Space IDs and page + * restrictions are cached for descendants within the batch. Space audiences + * are refreshed and persisted on demand using this source's credential. * * A page falls back to *its own* space's readers, never the union of every * configured space: a connector over two spaces must not let a reader of one @@ -621,35 +634,66 @@ async function resolveConfluenceAcls( accessToken: string, sourceConfig: Record, documents: readonly ExternalDocument[], - syncContext?: Record + syncContext?: Record, + aclContext?: ConnectorAclContext ): Promise> { + const selectionError = getSourceSelectionError(sourceConfig.spaceKey) + if (selectionError) throw new Error(selectionError) const cloudId = await resolveCloudId(accessToken, sourceConfig, syncContext) const spaceIdForKey = memoizeAsync((spaceKey: string) => resolveSpaceId(cloudId, accessToken, spaceKey) ) - const spacePrincipalsFor = memoizeAsync((spaceId: string) => - listSpaceReadPrincipals(cloudId, accessToken, spaceId) - ) const readRestriction = memoizeAsync((contentId: string) => getReadRestriction(cloudId, accessToken, contentId) ) + const spaceAudience = memoizeAsync(async (spaceId: string) => { + if (!aclContext) throw new Error('Confluence space membership persistence is unavailable') + const membership = await listConfluenceSpaceMembership( + CONFLUENCE_ACL_PROVIDER_ID, + cloudId, + accessToken, + spaceId + ) + await aclContext.persistGroupMembership({ + providerId: CONFLUENCE_ACL_PROVIDER_ID, + tenantId: cloudId, + group: membership.group, + memberTokens: membership.memberTokens, + }) + return membership.group.id + }) + /** The listing usually says where a page lives; anything it did not describe is asked. */ const locate = async (doc: ExternalDocument): Promise => { + if (isConfluenceAttachment(doc.externalId)) { + return locateConfluenceAttachment( + { + accessToken, + cloudId, + domain: normalizeConfluenceDomainHost(sourceConfig.domain as string), + syncContext, + }, + sourceConfig, + doc + ) + } const spaceKey = doc.metadata?.spaceKey const contentType = doc.metadata?.contentType if (typeof spaceKey === 'string' && spaceKey) { return { + id: doc.externalId, spaceId: await spaceIdForKey(spaceKey), contentType: typeof contentType === 'string' ? contentType : 'page', } } - return describeContent(cloudId, accessToken, doc.externalId) + const location = await describeContent(cloudId, accessToken, doc.externalId) + return location ? { id: doc.externalId, ...location } : null } /** One entry per page whose permissions this run could read in full. */ - const resolved = new Map() + const resolved = new Map() let unreadable = 0 await mapWithConcurrency(documents, ACL_CONCURRENCY, async (doc) => { const externalId = doc.externalId @@ -659,20 +703,19 @@ async function resolveConfluenceAcls( unreadable += 1 return } - const own = await readRestriction(externalId) + const own = await readRestriction(location.id) /** * Every ancestor restriction still applies when the page has its own. * A blog post has no ancestors to inherit from. */ const chain: ConfluenceRestriction[] = [own] if (location.contentType !== 'blogpost') { - for (const ancestorId of await listAncestorIds(cloudId, accessToken, externalId)) { + for (const ancestorId of await listAncestorIds(cloudId, accessToken, location.id)) { const restriction = await readRestriction(ancestorId) chain.push(restriction) } } - await spacePrincipalsFor(location.spaceId) - resolved.set(externalId, { spaceId: location.spaceId, chain }) + resolved.set(externalId, { spaceGroupId: await spaceAudience(location.spaceId), chain }) } catch (error) { unreadable += 1 logger.warn("Could not verify a page's permissions", { @@ -684,9 +727,9 @@ async function resolveConfluenceAcls( }) const acls: Record = {} - for (const [externalId, { spaceId, chain }] of resolved) { + for (const [externalId, { spaceGroupId, chain }] of resolved) { const result = confluencePageAcl({ - spacePrincipals: await spacePrincipalsFor(spaceId), + spacePrincipals: [{ kind: 'group', id: spaceGroupId }], restrictionChain: chain, providerId: CONFLUENCE_ACL_PROVIDER_ID, tenantId: cloudId, @@ -706,83 +749,108 @@ async function resolveConfluenceAcls( return acls } -export const confluenceConnector: ConnectorConfig = { - isCredentialInvalidError: (error) => - error instanceof Error && 'status' in error && error.status === 401, - ...confluenceConnectorMeta, - - listDocuments: async ( - accessToken: string, - sourceConfig: Record, - cursor?: string, - syncContext?: Record, - lastSyncAt?: Date - ): Promise => { - const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string) - const spaceKeys = parseMultiValue(sourceConfig.spaceKey) - const contentType = (sourceConfig.contentType as string) || 'page' - const labelFilter = (sourceConfig.labelFilter as string) || '' - const maxPages = sourceConfig.maxPages ? Number(sourceConfig.maxPages) : 0 - - if (spaceKeys.length === 0) { - throw new Error('At least one space key is required') - } - - const cloudId = await resolveCloudId(accessToken, sourceConfig, syncContext) +async function listParentDocuments( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record +): Promise { + const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string) + const allSpaces = isAllSourceItems(sourceConfig.spaceKey) + const spaceKeys = parseMultiValue(sourceConfig.spaceKey) + const contentType = (sourceConfig.contentType as string) || 'page' + const labelFilter = (sourceConfig.labelFilter as string) || '' + const maxPages = sourceConfig.maxPages ? Number(sourceConfig.maxPages) : 0 + + if (spaceKeys.length === 0) { + throw new Error('At least one space key is required') + } - /** - * Route through CQL when a label filter is set, when multiple spaces are - * selected, or when only recently modified content is wanted — the v2 - * `/spaces/{spaceId}/pages` endpoint is single-space only and cannot filter - * by modification time, but CQL natively supports `space in (...)` and - * `lastModified`. - */ - if (labelFilter.trim() || spaceKeys.length > 1 || lastSyncAt) { - return listSpaceBatchesViaCql( - cloudId, - accessToken, - domain, - spaceKeys, - contentType, - labelFilter, - maxPages, - cursor, - syncContext, - lastSyncAt - ) - } + const cloudId = await resolveCloudId(accessToken, sourceConfig, syncContext) - const spaceKey = spaceKeys[0] - let spaceId = syncContext?.spaceId as string | undefined - if (!spaceId) { - spaceId = await resolveSpaceId(cloudId, accessToken, spaceKey) - if (syncContext) syncContext.spaceId = spaceId - } + /** + * Route through CQL when a label filter is set, when multiple spaces are + * selected — the v2 space endpoint cannot apply those filters. + */ + if (allSpaces) { + return listDocumentsViaCql( + cloudId, + accessToken, + domain, + [], + contentType, + labelFilter, + maxPages, + cursor, + syncContext + ) + } + if (labelFilter.trim() || spaceKeys.length > 1) { + return listSpaceBatchesViaCql( + cloudId, + accessToken, + domain, + spaceKeys, + contentType, + labelFilter, + maxPages, + cursor, + syncContext + ) + } - if (contentType === 'all') { - return listAllContentTypes( - cloudId, - accessToken, - domain, - spaceId, - spaceKey, - maxPages, - cursor, - syncContext - ) - } + const spaceKey = spaceKeys[0] + let spaceId = syncContext?.spaceId as string | undefined + if (!spaceId) { + spaceId = await resolveSpaceId(cloudId, accessToken, spaceKey) + if (syncContext) syncContext.spaceId = spaceId + } - return listDocumentsV2( + if (contentType === 'all') { + return listAllContentTypes( cloudId, accessToken, domain, spaceId, spaceKey, - contentType, maxPages, cursor, syncContext ) + } + + return listDocumentsV2( + cloudId, + accessToken, + domain, + spaceId, + spaceKey, + contentType, + maxPages, + cursor, + syncContext + ) +} + +export const confluenceConnector: ConnectorConfig = { + isCredentialInvalidError: (error) => + error instanceof Error && 'status' in error && error.status === 401, + ...confluenceConnectorMeta, + + listDocuments: async (accessToken, sourceConfig, cursor, syncContext) => { + const selectionError = getSourceSelectionError(sourceConfig.spaceKey) + if (selectionError) throw new Error(selectionError) + const cloudId = await resolveCloudId(accessToken, sourceConfig, syncContext) + return listConfluenceAttachments({ + accessToken, + cloudId, + domain: normalizeConfluenceDomainHost(sourceConfig.domain as string), + cursor, + syncContext, + /** Attachment versions change independently of their parent pages. */ + listParents: (parentCursor, parentContext) => + listParentDocuments(accessToken, sourceConfig, parentCursor, parentContext), + }) }, getDocumentAcls: resolveConfluenceAcls, @@ -800,9 +868,19 @@ export const confluenceConnector: ConnectorConfig = { externalId: string, syncContext?: Record ): Promise => { + const selectionError = getSourceSelectionError(sourceConfig.spaceKey) + if (selectionError) throw new Error(selectionError) const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string) const cloudId = await resolveCloudId(accessToken, sourceConfig, syncContext) + if (isConfluenceAttachment(externalId)) { + return getConfluenceAttachment( + { accessToken, cloudId, domain, syncContext }, + sourceConfig, + externalId + ) + } + const scopedContent = usesPermissionScopedContent(syncContext) const bodyFormat = scopedContent ? 'storage' : 'view' let page: Record | null = null @@ -874,7 +952,10 @@ export const confluenceConnector: ConnectorConfig = { sourceConfig: Record, syncContext?: Record ): Promise<{ valid: boolean; error?: string }> => { + const selectionError = getSourceSelectionError(sourceConfig.spaceKey) + if (selectionError) return { valid: false, error: selectionError } const domain = sourceConfig.domain as string + const allSpaces = isAllSourceItems(sourceConfig.spaceKey) const spaceKeys = parseMultiValue(sourceConfig.spaceKey) if (!domain || spaceKeys.length === 0) { @@ -897,14 +978,14 @@ export const confluenceConnector: ConnectorConfig = { : VALIDATE_RETRY_OPTIONS const cloudId = await resolveCloudId(accessToken, sourceConfig, syncContext, retryOptions) let permissionSpaceId: string | undefined - for (const batch of spaceKeyBatches(spaceKeys)) { + for (const batch of allSpaces ? [[]] : spaceKeyBatches(spaceKeys)) { const remainingKeys = new Set(batch) const seenCursors = new Set() let cursor: string | undefined do { const params = new URLSearchParams() for (const key of batch) params.append('keys', key) - params.set('limit', String(batch.length)) + params.set('limit', String(allSpaces ? 1 : batch.length)) if (cursor) params.set('cursor', cursor) const response = await fetchWithRetry( `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?${params}`, @@ -923,7 +1004,7 @@ export const confluenceConnector: ConnectorConfig = { } for (const space of data.results) { if ( - remainingKeys.delete(space.key) && + (allSpaces || remainingKeys.delete(space.key)) && !permissionSpaceId && typeof space.id === 'string' ) { @@ -1042,7 +1123,8 @@ async function listDocumentsV2( } const data = await response.json() - const results = data.results || [] + if (!Array.isArray(data.results)) throw new Error('Confluence returned an invalid content page') + const results = data.results const allDocuments: ExternalDocument[] = (results as Record[]) .filter(isCurrentContent) @@ -1060,7 +1142,11 @@ async function listDocumentsV2( ) }) - const nextCursor = extractCursor((data._links as Record | undefined)?.next) + const next = (data._links as Record | undefined)?.next + const nextCursor = extractCursor(next) + if (next && (!nextCursor || nextCursor === cursor)) { + throw new Error('Confluence returned an invalid or repeated content continuation') + } const fetchedSoFar = (syncContext?.totalDocsFetched as number) ?? 0 const remaining = maxPages > 0 ? Math.max(0, maxPages - fetchedSoFar) : Number.POSITIVE_INFINITY @@ -1173,34 +1259,6 @@ async function listAllContentTypes( return results } -/** - * The CQL clause selecting content modified since a watermark. CQL's `now()` - * takes a relative offset and evaluates on the server, which sidesteps the - * timezone the endpoint would otherwise assume for an absolute timestamp; the - * offset rounds up to the next whole minute so nothing at the edge is missed. - */ -export function buildLastModifiedClause(lastSyncAt: Date, now: Date): string { - const minutes = Math.max(1, Math.ceil((now.getTime() - lastSyncAt.getTime()) / 60_000)) - return `lastModified >= now("-${minutes}m")` -} - -/** - * The `lastModified` clause every page of one listing shares. The clause is a - * window relative to the server clock, so recomputing it on a later page that - * crosses a minute boundary would pair the cursor `_links.next` issued with a - * query it was not issued for; the first page fixes it for the run. - */ -export function resolveLastModifiedClause( - lastSyncAt: Date, - syncContext: Record | undefined -): string { - const fixed = syncContext?.cqlLastModifiedClause - if (typeof fixed === 'string') return fixed - const clause = buildLastModifiedClause(lastSyncAt, new Date()) - if (syncContext) syncContext.cqlLastModifiedClause = clause - return clause -} - /** * Page size for CQL search. The endpoint defaults to 25 and documents no hard * maximum, so this stays conservatively below the fixed system limits it warns @@ -1218,8 +1276,7 @@ async function listSpaceBatchesViaCql( labelFilter: string, maxPages: number, cursor?: string, - syncContext: Record = {}, - lastSyncAt?: Date + syncContext: Record = {} ): Promise { const batches = spaceKeyBatches(spaceKeys) if (batches.length === 1) { @@ -1232,8 +1289,7 @@ async function listSpaceBatchesViaCql( labelFilter, maxPages, cursor, - syncContext, - lastSyncAt + syncContext ) } @@ -1267,8 +1323,7 @@ async function listSpaceBatchesViaCql( labelFilter, maxPages, providerCursor, - syncContext, - lastSyncAt + syncContext ) const hasMoreBatches = batchIndex + 1 < batches.length if (maxPages > 0 && Number(syncContext.totalDocsFetched) >= maxPages) { @@ -1289,7 +1344,7 @@ async function listSpaceBatchesViaCql( } /** - * Lists documents using CQL search via the v1 API (used when label filtering is enabled). + * Lists parents through CQL for all-space, multi-space, and label-filtered sources. */ async function listDocumentsViaCql( cloudId: string, @@ -1300,29 +1355,25 @@ async function listDocumentsViaCql( labelFilter: string, maxPages: number, cursor?: string, - syncContext?: Record, - lastSyncAt?: Date + syncContext?: Record ): Promise { const labels = labelFilter .split(',') .map((l) => l.trim()) .filter(Boolean) - // Build CQL query - let cql = buildSpaceClause(spaceKeys) + let cql = spaceKeys.length > 0 ? `${buildSpaceClause(spaceKeys)} AND ` : '' if (contentType === 'blogpost') { - cql += ' AND type="blogpost"' + cql += 'type="blogpost"' } else if (contentType === 'all') { /** - * An unconstrained CQL search matches every content type the index holds — - * attachments, comments, space descriptions and user profiles included — none - * of which `getDocument` can resolve through the page/blogpost endpoints. "All - * content" means both indexable content types, not literally everything. + * Restrict parent discovery to pages and blog posts; supported attachments + * are listed separately beneath these parents. */ - cql += ' AND type in ("page","blogpost")' + cql += 'type in ("page","blogpost")' } else { - cql += ' AND type="page"' + cql += 'type="page"' } if (labels.length === 1) { @@ -1332,8 +1383,6 @@ async function listDocumentsViaCql( cql += ` AND label in (${labelList})` } - if (lastSyncAt) cql += ` AND ${resolveLastModifiedClause(lastSyncAt, syncContext)}` - const fetchedSoFar = (syncContext?.totalDocsFetched as number) ?? 0 const remaining = maxPages > 0 ? maxPages - fetchedSoFar : Number.POSITIVE_INFINITY @@ -1381,7 +1430,8 @@ async function listDocumentsViaCql( } const data = await response.json() - const results = data.results || [] + if (!Array.isArray(data.results)) throw new Error('Confluence returned an invalid search page') + const results = data.results const allDocuments: ExternalDocument[] = (results as Record[]) .filter(isCurrentContent) @@ -1396,7 +1446,11 @@ async function listDocumentsViaCql( allDocuments.length > remaining ? allDocuments.slice(0, remaining) : allDocuments const trimmedByCap = documents.length < allDocuments.length - const nextCursor = extractCursor((data._links as Record | undefined)?.next) + const next = (data._links as Record | undefined)?.next + const nextCursor = extractCursor(next) + if (next && (!nextCursor || nextCursor === cursor)) { + throw new Error('Confluence returned an invalid or repeated search continuation') + } const totalFetched = fetchedSoFar + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched diff --git a/apps/sim/connectors/confluence/meta.ts b/apps/sim/connectors/confluence/meta.ts index b7b3f0782b3..16cebaceb8e 100644 --- a/apps/sim/connectors/confluence/meta.ts +++ b/apps/sim/connectors/confluence/meta.ts @@ -1,4 +1,5 @@ import { ConfluenceIcon } from '@/components/icons' +import { ALL_SOURCE_ITEMS } from '@/connectors/selection' import type { ConnectorMeta } from '@/connectors/types' export const confluenceConnectorMeta: ConnectorMeta = { @@ -14,6 +15,7 @@ export const confluenceConnectorMeta: ConnectorMeta = { mode: 'oauth', provider: 'confluence', adminCredentialType: 'service_account', + /** Attachment access is optional so older credentials can keep syncing parent content. */ requiredScopes: [ 'read:confluence-content.all', 'read:page:confluence', @@ -28,6 +30,7 @@ export const confluenceConnectorMeta: ConnectorMeta = { 'read:confluence-content.all', 'read:page:confluence', 'read:blogpost:confluence', + 'read:attachment:confluence', 'read:space:confluence', 'read:label:confluence', 'search:confluence', @@ -78,6 +81,7 @@ export const confluenceConnectorMeta: ConnectorMeta = { mode: 'basic', multi: true, allowSelectAll: true, + selectAllValue: ALL_SOURCE_ITEMS, preserveValueOnModeChange: true, dependsOn: ['domain'], placeholder: 'Select one or more spaces', diff --git a/apps/sim/connectors/confluence/permissions.test.ts b/apps/sim/connectors/confluence/permissions.test.ts index 925230bdc21..163b16aa564 100644 --- a/apps/sim/connectors/confluence/permissions.test.ts +++ b/apps/sim/connectors/confluence/permissions.test.ts @@ -2,12 +2,13 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' -import { MAX_ACL_TOKENS } from '@/lib/knowledge/access/tokens' import { getReadRestriction, listAncestorIds, + listConfluenceSpaceMembership, listGroupMemberTokens, listSpaceReadPrincipals, + openConfluenceDirectory, } from '@/connectors/confluence/permissions' const mockFetch = vi.fn() @@ -26,6 +27,63 @@ beforeEach(() => { }) describe('listSpaceReadPrincipals', () => { + it('retains more than 5,000 readers in a space audience without expanding document ACLs', async () => { + let page = 0 + mockFetch.mockImplementation(async () => { + const offset = page++ * 250 + return jsonResponse({ + results: Array.from({ length: 250 }, (_, index) => ({ + principal: { type: 'user', id: `reader-${offset + index}` }, + operation: { key: 'read', targetType: 'space' }, + })), + ...(page < 24 ? { _links: { next: `?cursor=${page}` } } : {}), + }) + }) + const membership = await listConfluenceSpaceMembership('confluence', CLOUD, 'token', '123') + expect(membership.complete).toBe(true) + expect(membership.memberTokens).toHaveLength(6000) + expect(membership.memberTokens[5999]).toBe('s:confluence:-:reader-5999') + expect(mockFetch).toHaveBeenCalledTimes(24) + }) + + it('stores native reader groups without flattening their membership', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ + results: [ + { + principal: { type: 'group', id: 'engineering' }, + operation: { key: 'read', targetType: 'space' }, + }, + ], + }) + ) + await expect( + listConfluenceSpaceMembership('confluence', CLOUD, 'token', '123') + ).resolves.toEqual({ + group: { id: 'space-readers:123' }, + memberTokens: ['g:confluence:cloud-1:engineering'], + complete: true, + }) + }) + + it('lists only site-native groups without enumerating unrelated visible spaces', async () => { + mockFetch.mockResolvedValueOnce(jsonResponse({ results: [{ id: 'engineering' }] })) + const directory = openConfluenceDirectory('confluence', CLOUD, 'token') + await expect(directory.listGroups()).resolves.toEqual([{ id: 'engineering' }]) + expect(mockFetch).toHaveBeenCalledOnce() + expect(mockFetch.mock.calls[0][0]).toContain('/rest/api/group?') + }) + + it.each(['space-readers:123', ' SPACE-READERS:123 '])( + 'rejects native groups in the reserved namespace: %s', + async (id) => { + mockFetch.mockResolvedValueOnce(jsonResponse({ results: [{ id }] })) + await expect( + openConfluenceDirectory('confluence', CLOUD, 'token').listGroups() + ).rejects.toThrow('invalid group ID') + } + ) + it('keeps only the permission that grants reading the space', () => { mockFetch.mockResolvedValueOnce( jsonResponse({ @@ -419,9 +477,9 @@ describe('listSpaceReadPrincipals', () => { }) ) await expect(listSpaceReadPrincipals(CLOUD, 'token', 'space')).rejects.toThrow( - 'document permission limit' + 'directory capacity' ) - expect(mockFetch).toHaveBeenCalledTimes(Math.floor(MAX_ACL_TOKENS / 250) + 1) + expect(mockFetch).toHaveBeenCalledTimes(401) }) it('rejects an oversized permission response before accepting its readers', async () => { diff --git a/apps/sim/connectors/confluence/permissions.ts b/apps/sim/connectors/confluence/permissions.ts index 5e8e8f1470e..af9c8642281 100644 --- a/apps/sim/connectors/confluence/permissions.ts +++ b/apps/sim/connectors/confluence/permissions.ts @@ -5,9 +5,11 @@ import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { type ConfluencePrincipal, type ConfluenceRestriction, + confluenceSpaceGroupId, confluenceSubjectToken, } from '@/lib/knowledge/access/confluence-permissions' -import { MAX_ACL_TOKENS } from '@/lib/knowledge/access/tokens' +import { CONFLUENCE_SPACE_GROUP_PREFIX } from '@/lib/knowledge/access/confluence-space-groups' +import { canonicalGroupId, groupToken } from '@/lib/knowledge/access/tokens' import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { extractCursor } from '@/connectors/confluence/cursor' @@ -26,10 +28,20 @@ const GROUP_PAGE_SIZE = 200 const MAX_PAGES = 100 const MAX_PERMISSION_PAGES = 1000 const PERMISSION_RESPONSE_MAX_BYTES = 1024 * 1024 +const MAX_SPACE_READERS = 100_000 +const MAX_SPACE_READER_BYTES = 16 * 1024 * 1024 const MAX_PERMISSION_CURSOR_LENGTH = 8192 const PERMISSION_COLLECTION_TIMEOUT_MS = 5 * 60 * 1000 const PREFLIGHT_RESPONSE_MAX_BYTES = 256 * 1024 +function validateNativeGroupId(id: string): string { + const canonical = canonicalGroupId(id) + if (!canonical || canonical.startsWith(CONFLUENCE_SPACE_GROUP_PREFIX)) { + throw new Error('Confluence returned an invalid group ID') + } + return id +} + function apiBase(cloudId: string): string { return `https://api.atlassian.com/ex/confluence/${cloudId}/wiki` } @@ -313,11 +325,18 @@ export async function listSpaceReadPrincipals( if (!id) { unmapped += 1 } else if (type === 'user' || type === 'group') { + if (type === 'group') validateNativeGroupId(id) const key = `${type}:${id}` if (!principals.has(key)) { principalBytes += Buffer.byteLength(key, 'utf8') - if (principals.size >= MAX_ACL_TOKENS || principalBytes > PERMISSION_RESPONSE_MAX_BYTES) { - throw new Error('Confluence space readers exceeded the document permission limit') + if (principals.size >= MAX_SPACE_READERS || principalBytes > MAX_SPACE_READER_BYTES) { + logger.warn('Confluence space audience exceeded the directory capacity', { + cloudId, + spaceId, + principals: principals.size, + principalBytes, + }) + throw new Error('Confluence space readers exceeded the directory capacity') } principals.set(key, { kind: type, id }) } @@ -477,6 +496,7 @@ export async function getReadRestriction( } for (const group of groups) { if (!group.id) throw new Error('Confluence read restriction is missing a group id') + validateNativeGroupId(group.id) principals.set(`group:${group.id}`, { kind: 'group', id: group.id }) } const nextStarts = [ @@ -565,7 +585,7 @@ async function listSiteGroups( ) const groups: ConnectorDirectoryGroup[] = [] for (const group of raw) { - if (group.id) groups.push({ id: group.id }) + groups.push({ id: validateNativeGroupId(group.id ?? '') }) } return groups } @@ -602,3 +622,21 @@ export function openConfluenceDirectory( listGroupMembers: (group) => listGroupMemberTokens(cloudId, accessToken, group), } } + +/** Only spaces encountered by this credential's content crawl need an audience refresh. */ +export async function listConfluenceSpaceMembership( + providerId: string, + cloudId: string, + accessToken: string, + spaceId: string +): Promise { + const group = { id: confluenceSpaceGroupId(spaceId) } + const principals = await listSpaceReadPrincipals(cloudId, accessToken, spaceId) + const memberTokens = principals.map((principal) => { + if (principal.kind === 'user') return confluenceSubjectToken(principal.id) + const token = groupToken({ providerId, tenantId: cloudId, groupId: principal.id }) + if (!token) throw new Error('Confluence returned an invalid reader group') + return token + }) + return { group, memberTokens, complete: true } +} diff --git a/apps/sim/connectors/jira/jira.test.ts b/apps/sim/connectors/jira/jira.test.ts index a2c08e7a0ff..2a3ffe7a2cc 100644 --- a/apps/sim/connectors/jira/jira.test.ts +++ b/apps/sim/connectors/jira/jira.test.ts @@ -61,6 +61,31 @@ afterEach(() => { }) describe('Jira Search member documents', () => { + it('resolves All through the current credential across pages and future syncs', async () => { + fetchMock + .mockResolvedValueOnce(json({ issues: [issue()], nextPageToken: 'page-two', isLast: false })) + .mockResolvedValueOnce( + json({ issues: [issue('20001', { project: { id: '20000', key: 'HR' } })], isLast: true }) + ) + .mockResolvedValueOnce( + json({ issues: [issue('30001', { project: { id: '30000', key: 'NEW' } })], isLast: true }) + ) + const config = { ...SOURCE, projectKey: ['*'], jql: 'status = Open' } + const context = { ...MEMBERS } + const first = await jiraConnector.listDocuments('token', config, undefined, context) + const second = await jiraConnector.listDocuments('token', config, first.nextCursor, context) + const nextSync = await jiraConnector.listDocuments('token', config, undefined, { ...MEMBERS }) + expect(first.hasMore).toBe(true) + expect(second.documents[0].sourceUrl).toContain('ENG-20001') + expect(nextSync.documents[0].sourceUrl).toContain('ENG-30001') + expect(second.hasMore).toBe(false) + const urls = fetchMock.mock.calls.map(([input]) => new URL(String(input))) + expect(urls.map((url) => url.searchParams.get('jql'))).toEqual( + Array(3).fill('project IS NOT EMPTY AND (status = Open) ORDER BY updated DESC') + ) + expect(urls[1].searchParams.get('nextPageToken')).toBe('page-two') + }) + it('offers managed member Search with canonical project setup and no central ACL mode', () => { expect(jiraConnectorMeta.search).toBe(true) expect(jiraConnectorMeta.permissionScopedListing?.capFieldIds).toEqual(['maxIssues']) diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index 52c0a766a89..781f1f566b3 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -10,6 +10,7 @@ import { import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jiraConnectorMeta } from '@/connectors/jira/meta' +import { getSourceSelectionError, isAllSourceItems } from '@/connectors/selection' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { computeContentHash, @@ -144,6 +145,7 @@ function getMaxIssues(sourceConfig: Record): number { * Each key is escaped for inclusion in a JQL double-quoted string. */ function buildProjectClause(projectKeys: string[]): string { + if (isAllSourceItems(projectKeys)) return 'project IS NOT EMPTY' const escapeKey = (key: string) => key.replace(/\\/g, '\\\\').replace(/"/g, '\\"') if (projectKeys.length === 1) { return `project = "${escapeKey(projectKeys[0])}"` @@ -270,8 +272,11 @@ async function issueToMemberDocument( /** JQL can refine the configured projects but must not expand their scope. */ function isInConfiguredProject(issue: JiraIssue, projectKeys: string[]): boolean { const project = issue.fields.project - return projectKeys.some( - (value) => value === project.id || value.toUpperCase() === project.key.toUpperCase() + return ( + isAllSourceItems(projectKeys) || + projectKeys.some( + (value) => value === project.id || value.toUpperCase() === project.key.toUpperCase() + ) ) } @@ -298,6 +303,8 @@ export const jiraConnector: ConnectorConfig = { cursor?: string, syncContext?: Record ): Promise => { + const selectionError = getSourceSelectionError(sourceConfig.projectKey) + if (selectionError) throw new Error(selectionError) const domain = sourceConfig.domain as string const siteUrl = normalizeAtlassianSiteUrl(domain) const projectKeys = parseMultiValue(sourceConfig.projectKey) @@ -522,6 +529,8 @@ export const jiraConnector: ConnectorConfig = { externalId: string, syncContext?: Record ): Promise => { + const selectionError = getSourceSelectionError(sourceConfig.projectKey) + if (selectionError) throw new Error(selectionError) const domain = sourceConfig.domain as string const siteUrl = normalizeAtlassianSiteUrl(domain) const cloudId = await resolveCloudId(accessToken, domain, syncContext) @@ -568,6 +577,8 @@ export const jiraConnector: ConnectorConfig = { sourceConfig: Record, syncContext?: Record ): Promise<{ valid: boolean; error?: string }> => { + const selectionError = getSourceSelectionError(sourceConfig.projectKey) + if (selectionError) return { valid: false, error: selectionError } const domain = sourceConfig.domain as string const projectKeys = parseMultiValue(sourceConfig.projectKey) diff --git a/apps/sim/connectors/jira/meta.ts b/apps/sim/connectors/jira/meta.ts index d99f48cf0d6..247a4b8bd0e 100644 --- a/apps/sim/connectors/jira/meta.ts +++ b/apps/sim/connectors/jira/meta.ts @@ -1,4 +1,5 @@ import { JiraIcon } from '@/components/icons' +import { ALL_SOURCE_ITEMS } from '@/connectors/selection' import type { ConnectorMeta } from '@/connectors/types' export const jiraConnectorMeta: ConnectorMeta = { @@ -30,6 +31,7 @@ export const jiraConnectorMeta: ConnectorMeta = { mode: 'basic', multi: true, allowSelectAll: true, + selectAllValue: ALL_SOURCE_ITEMS, preserveValueOnModeChange: true, dependsOn: ['domain'], placeholder: 'Select one or more projects', diff --git a/apps/sim/connectors/selection.ts b/apps/sim/connectors/selection.ts new file mode 100644 index 00000000000..ac8da6b8bb9 --- /dev/null +++ b/apps/sim/connectors/selection.ts @@ -0,0 +1,34 @@ +import type { ConnectorMeta } from '@/connectors/types' +import { parseMultiValue } from '@/connectors/utils' + +/** Persisted scope marker; providers resolve the accessible set during each listing. */ +export const ALL_SOURCE_ITEMS = '*' + +export function isAllSourceItems(value: unknown): boolean { + const items = parseMultiValue(value) + return items.length === 1 && items[0] === ALL_SOURCE_ITEMS +} + +export function getSourceSelectionError( + value: unknown, + allValue = ALL_SOURCE_ITEMS +): string | undefined { + const items = parseMultiValue(value) + if (items.length > 1 && items.includes(allValue)) { + return `Use "${allValue}" by itself for All, or remove it to select individual items.` + } +} + +export function findSourceSelectionError( + connector: Pick, + sourceConfig: Record +): string | undefined { + for (const field of connector.configFields) { + if (!field.selectAllValue) continue + const error = getSourceSelectionError( + sourceConfig[field.canonicalParamId ?? field.id], + field.selectAllValue + ) + if (error) return error + } +} diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts index 5c5bc3f49b0..7014bdd8586 100644 --- a/apps/sim/connectors/types.ts +++ b/apps/sim/connectors/types.ts @@ -69,7 +69,7 @@ export interface ConnectorDirectoryGroup { export interface ConnectorDirectoryMembership { group: ConnectorDirectoryGroup - /** Canonical u:email or provider-attested s: identity tokens, with nested groups flattened. */ + /** Canonical identities; Confluence space audiences may also name same-site native groups. */ memberTokens: string[] /** * False when the walk could not be completed. A partial membership must never @@ -79,6 +79,19 @@ export interface ConnectorDirectoryMembership { complete: boolean } +/** A complete source audience observed while resolving a bounded batch of document ACLs. */ +export interface ConnectorAclGroupMembership { + providerId: string + tenantId: string + group: ConnectorDirectoryGroup + memberTokens: string[] +} + +/** Persistence capabilities bound by the engine to the canonical resource owner and sync lease. */ +export interface ConnectorAclContext { + persistGroupMembership: (membership: ConnectorAclGroupMembership) => Promise +} + /** * One directory, opened for the length of a sync. Implementations throw rather * than returning a partial listing — a truncated directory read as complete @@ -330,8 +343,10 @@ export interface ConnectorConfigField { * Connector handlers receive `string | string[]` and should normalize via `parseMultiValue`. */ multi?: boolean - /** Offers explicit bulk selection of the complete, bounded provider list. */ + /** Offers selection of all items, using selectAllValue when configured. */ allowSelectAll?: boolean + /** Stores a provider-supported scope marker instead of the currently loaded option IDs. */ + selectAllValue?: string } /** @@ -583,7 +598,8 @@ export interface ConnectorConfig extends ConnectorMeta { accessToken: string, sourceConfig: Record, documents: readonly ExternalDocument[], - syncContext?: Record + syncContext?: Record, + aclContext?: ConnectorAclContext ) => Promise> /** Map source metadata to semantic tag keys (translated to slots by the sync engine) */ diff --git a/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts b/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts index ea7b12dc25f..3cb48da93be 100644 --- a/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts @@ -23,7 +23,10 @@ import { } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { groupToken, subjectToken } from '@/lib/knowledge/access/tokens' import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { syncExternalDirectoryGroups } from '@/lib/knowledge/connectors/external-group-sync' +import { + persistExternalGroupMembership, + syncExternalDirectoryGroups, +} from '@/lib/knowledge/connectors/external-group-sync' describe('Confluence identities with hidden directory email', () => { const ids = createKnowledgeAclFixtureIds() @@ -45,6 +48,7 @@ describe('Confluence identities with hidden directory email', () => { const documents = [ { id: generateId(), acl: [group] }, { id: generateId(), acl: [sourceSubject] }, + { id: generateId(), acl: ['g:confluence:fixture-cloud:space-readers:123'] }, ] const principal: Principal = { kind: 'session', @@ -142,6 +146,22 @@ describe('Confluence identities with hidden directory email', () => { }), }, }) + await db.transaction((tx) => + persistExternalGroupMembership( + { + workspaceId: ids.workspaceId, + providerId: 'confluence', + tenantId: 'fixture-cloud', + group: { id: 'space-readers:123' }, + memberTokens: [ + group, + ...Array.from({ length: 6000 }, (_, index) => `s:confluence:-:other-${index}`), + ], + observedAt: new Date(), + }, + tx + ) + ) await db.insert(document).values( documents.map((fixture) => ({ id: fixture.id, diff --git a/apps/sim/lib/knowledge/__integration__/directory-sync.integration.ts b/apps/sim/lib/knowledge/__integration__/directory-sync.integration.ts index 7ced44aa61e..3ea03a7024d 100644 --- a/apps/sim/lib/knowledge/__integration__/directory-sync.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/directory-sync.integration.ts @@ -22,6 +22,7 @@ const fixture = vi.hoisted(() => ({ listGroups: vi.fn(), members: vi.fn(), listDocuments: vi.fn(), + acls: vi.fn(), enqueue: vi.fn(async (_type: string, _payload: unknown) => 'job'), })) vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: () => null })) @@ -35,6 +36,7 @@ vi.mock('@/connectors/registry.server', () => ({ name: 'Directory fixture', auth: { mode: 'apiKey', optional: true }, listDocuments: fixture.listDocuments, + getDocumentAcls: fixture.acls, getDocument: async () => { throw new Error('Unexpected hydration') }, @@ -57,13 +59,23 @@ import { import { resolveUserKnowledgeAccessScope } from '@/lib/knowledge/access/scope' import { groupToken } from '@/lib/knowledge/access/tokens' import { + persistExternalGroupMembership, refreshConnectorDirectory, syncExternalDirectoryGroups, } from '@/lib/knowledge/connectors/external-group-sync' +import { + beginListingCheckpoint, + listingFingerprint, +} from '@/lib/knowledge/connectors/listing-checkpoint' import { executeSync } from '@/lib/knowledge/connectors/sync-engine' import { GET as scheduleDirectories } from '@/app/api/knowledge/connectors/directory-sync/route' import { executeDirectorySyncJob } from '@/background/knowledge-connector-directory-sync' -import type { ConnectorDirectory, ConnectorDirectoryGroup } from '@/connectors/types' +import type { + ConnectorAclContext, + ConnectorDirectory, + ConnectorDirectoryGroup, + ExternalDocument, +} from '@/connectors/types' describe('directory failure visibility in PostgreSQL', () => { const ids = createKnowledgeAclFixtureIds() @@ -92,6 +104,7 @@ describe('directory failure visibility in PostgreSQL', () => { complete: true, })) fixture.listDocuments.mockReset().mockResolvedValue({ documents: [], hasMore: false }) + fixture.acls.mockReset().mockResolvedValue({}) await db .update(knowledgeConnector) .set({ @@ -161,6 +174,174 @@ describe('directory failure visibility in PostgreSQL', () => { return { promise, resolve } } + it('keeps disjoint credential audiences independent of native-directory freshness and pruning', async () => { + const directory = { ...directoryFixture(), providerId: 'confluence' } + const persistAudience = (spaceId: string, subject: string) => + db.transaction((tx) => + persistExternalGroupMembership( + { + workspaceId: ids.workspaceId, + providerId: directory.providerId, + tenantId: directory.tenantId, + group: { id: `space-readers:${spaceId}` }, + memberTokens: [subject], + observedAt: new Date(), + }, + tx + ) + ) + await syncExternalDirectoryGroups({ workspaceId: ids.workspaceId, directory }) + await persistAudience('1', 's:confluence:-:alice') + expect( + await syncExternalDirectoryGroups({ workspaceId: ids.workspaceId, directory }) + ).toMatchObject({ skipped: true }) + await persistAudience('2', 's:confluence:-:bob') + directory.listGroups = async () => [] + expect( + await syncExternalDirectoryGroups({ workspaceId: ids.workspaceId, directory, force: true }) + ).toMatchObject({ pruned: 1 }) + const groups = await db.select().from(knowledgeExternalGroup).where(groupsWhere(directory)) + expect(groups.map((group) => group.externalGroupId).sort()).toEqual([ + 'space-readers:1', + 'space-readers:2', + ]) + const memberships = await db + .select() + .from(knowledgeExternalGroupMember) + .where( + inArray( + knowledgeExternalGroupMember.groupId, + groups.map((group) => group.id) + ) + ) + expect(memberships.map((member) => member.subjectToken).sort()).toEqual([ + 's:confluence:-:alice', + 's:confluence:-:bob', + ]) + }) + + it('does not regrant an older audience when its delayed response arrives after a revocation', async () => { + const tenantId = generateId() + const observation = { + workspaceId: ids.workspaceId, + providerId: 'confluence', + tenantId, + group: { id: 'space-readers:1' }, + } + const newer = new Date() + await db.transaction((tx) => + persistExternalGroupMembership({ ...observation, memberTokens: [], observedAt: newer }, tx) + ) + await db.transaction((tx) => + persistExternalGroupMembership( + { + ...observation, + memberTokens: ['s:confluence:-:alice'], + observedAt: new Date(newer.getTime() - 60_000), + }, + tx + ) + ) + const [group] = await db + .select() + .from(knowledgeExternalGroup) + .where(eq(knowledgeExternalGroup.tenantId, tenantId)) + expect(group.lastSyncedAt).toEqual(newer) + expect( + await db + .select() + .from(knowledgeExternalGroupMember) + .where(eq(knowledgeExternalGroupMember.groupId, group.id)) + ).toEqual([]) + }) + + it('refreshes audience evidence and revocations when an old content generation resumes', async () => { + const pages = ['audience-first', 'audience-second'] + await db.insert(document).values( + pages.map((externalId) => ({ + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + connectorId: ids.connectorId, + externalId, + filename: externalId, + fileUrl: 'data:text/plain,fixture', + fileSize: 7, + mimeType: 'text/plain', + contentHash: `hash-${externalId}`, + processingStatus: 'completed', + acl: [], + })) + ) + const connector = await source() + const checkpoint = beginListingCheckpoint({ + fingerprint: listingFingerprint({ + connectorType: connector.connectorType, + credentialId: connector.credentialId, + encryptedApiKey: connector.encryptedApiKey, + sourceConfig: connector.sourceConfig, + accessMode: connector.accessMode, + }), + generationId: generateId(), + startedAt: old, + }) + await db + .update(knowledgeConnector) + .set({ listingCheckpoint: checkpoint }) + .where(eq(knowledgeConnector.id, ids.connectorId)) + fixture.listDocuments.mockImplementation(async (_token, _config, cursor) => { + const externalId = cursor ? pages[1] : pages[0] + return { + documents: [ + { + externalId, + title: externalId, + content: 'fixture', + contentHash: `hash-${externalId}`, + mimeType: 'text/plain', + }, + ], + hasMore: !cursor, + nextCursor: cursor ? undefined : 'second-page', + } + }) + const tenantId = generateId() + fixture.acls.mockImplementation( + async ( + _token, + _config, + documents: ExternalDocument[], + _syncContext, + context: ConnectorAclContext + ) => { + await context.persistGroupMembership({ + providerId: 'confluence', + tenantId, + group: { id: 'space-readers:1' }, + memberTokens: documents[0].externalId === pages[0] ? ['s:confluence:-:alice'] : [], + }) + return Object.fromEntries( + documents.map((doc) => [doc.externalId, [`g:confluence:${tenantId}:space-readers:1`]]) + ) + } + ) + const before = Date.now() + expect(await executeSync(ids.connectorId, { billingAttribution: billing })).toMatchObject({ + docsUnchanged: 2, + }) + expect(fixture.acls).toHaveBeenCalledTimes(2) + const [group] = await db + .select() + .from(knowledgeExternalGroup) + .where(eq(knowledgeExternalGroup.tenantId, tenantId)) + expect(group.lastSyncedAt!.getTime()).toBeGreaterThanOrEqual(before - 1000) + expect( + await db + .select() + .from(knowledgeExternalGroupMember) + .where(eq(knowledgeExternalGroupMember.groupId, group.id)) + ).toEqual([]) + }) + it('advances past the first 200 directories, including microsecond timestamps and competing ticks', async () => { const connectors = Array.from({ length: 201 }, () => ({ id: generateId(), @@ -425,7 +606,9 @@ describe('directory failure visibility in PostgreSQL', () => { })) await expect( executeDirectorySyncJob({ connectorId: ids.connectorId, requestId: generateId() }) - ).rejects.toThrow('Directory refresh failed: 1 group memberships could not be refreshed') + ).rejects.toThrow( + 'Directory permission sync failed. Group membership could not be fully verified.' + ) const [group] = await db .select() .from(knowledgeExternalGroup) @@ -458,7 +641,9 @@ describe('directory failure visibility in PostgreSQL', () => { it('reports a directory failure after an empty content crawl and recovers on a later sync', async () => { fixture.listGroups.mockRejectedValue(new Error('Directory API HTTP 403')) const result = await executeSync(ids.connectorId, { billingAttribution: billing }) - expect(result.error).toBe('Directory refresh failed: Directory API HTTP 403') + expect(result.error).toBe( + 'Directory permission sync failed. Group membership could not be fully verified.' + ) expect(fixture.listDocuments).toHaveBeenCalledOnce() const failed = await source() expect(failed.status).toBe('error') @@ -528,7 +713,7 @@ describe('directory failure visibility in PostgreSQL', () => { const result = await executeSync(ids.connectorId, { billingAttribution: billing }) expect(result).toMatchObject({ docsUnchanged: 2, - error: 'Directory refresh failed: Directory unavailable', + error: 'Directory permission sync failed. Group membership could not be fully verified.', }) expect(fixture.listDocuments).toHaveBeenCalledTimes(2) const indexed = await db @@ -547,10 +732,10 @@ describe('directory failure visibility in PostgreSQL', () => { fixture.listGroups.mockRejectedValue(new Error('Directory unavailable')) expect( (await executeSync(ids.connectorId, { billingAttribution: billing, fullSync: true })).error - ).toContain('Directory unavailable') + ).toContain('Directory permission sync failed') fixture.listGroups.mockClear() expect((await executeSync(ids.connectorId, { billingAttribution: billing })).error).toContain( - 'Directory unavailable' + 'Directory permission sync failed' ) expect(fixture.listGroups).toHaveBeenCalledOnce() fixture.listGroups.mockResolvedValue(ids.groups.map((id) => ({ id }))) diff --git a/apps/sim/lib/knowledge/access/confluence-permissions.ts b/apps/sim/lib/knowledge/access/confluence-permissions.ts index 094aa9687ff..a70412bacdf 100644 --- a/apps/sim/lib/knowledge/access/confluence-permissions.ts +++ b/apps/sim/lib/knowledge/access/confluence-permissions.ts @@ -1,3 +1,4 @@ +import { CONFLUENCE_SPACE_GROUP_PREFIX } from '@/lib/knowledge/access/confluence-space-groups' import { groupToken, sortAccessTokens, subjectToken } from '@/lib/knowledge/access/tokens' import { LINK_ACCESS_TOKEN } from '@/lib/knowledge/access/types' @@ -7,6 +8,12 @@ export interface ConfluencePrincipal { id: string } +/** Space audiences use a namespace rejected at native group ingestion. */ +export function confluenceSpaceGroupId(spaceId: string): string { + if (!/^\d+$/.test(spaceId)) throw new Error('Confluence returned an invalid space ID') + return `${CONFLUENCE_SPACE_GROUP_PREFIX}${spaceId}` +} + /** Atlassian account IDs are global, matching the managed OAuth /me identity. */ export function confluenceSubjectToken(accountId: string): string { return subjectToken({ diff --git a/apps/sim/lib/knowledge/access/confluence-space-groups.ts b/apps/sim/lib/knowledge/access/confluence-space-groups.ts new file mode 100644 index 00000000000..86013456592 --- /dev/null +++ b/apps/sim/lib/knowledge/access/confluence-space-groups.ts @@ -0,0 +1,6 @@ +/** Reserved for source audiences; native Confluence group ingestion rejects this namespace. */ +export const CONFLUENCE_SPACE_GROUP_PREFIX = 'space-readers:' + +export function isConfluenceSpaceGroupId(groupId: string): boolean { + return /^space-readers:\d+$/.test(groupId) +} diff --git a/apps/sim/lib/knowledge/access/group-membership.integration.ts b/apps/sim/lib/knowledge/access/group-membership.integration.ts new file mode 100644 index 00000000000..b3c39d79900 --- /dev/null +++ b/apps/sim/lib/knowledge/access/group-membership.integration.ts @@ -0,0 +1,79 @@ +import { drizzle } from 'drizzle-orm/postgres-js' +import type postgres from 'postgres' +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { createEnterpriseSearchMigrationFixture } from '@/lib/knowledge/__integration__/migration-fixture' +import { parentGroupTokensQuery } from '@/lib/knowledge/access/group-membership' + +describe('Confluence space audience access in PostgreSQL', () => { + let fixture: Awaited> + let client: ReturnType + const cutoff = new Date('2026-09-17T00:00:00Z') + const scope = { kind: 'organization', organizationId: 'org' } as const + const token = (id: string) => `g:confluence:cloud:${id}` + + beforeAll(async () => { + fixture = await createEnterpriseSearchMigrationFixture( + process.env.KNOWLEDGE_ACL_TEST_DATABASE_URL! + ) + client = fixture.client + await fixture.migrate() + await client`INSERT INTO organization(id) VALUES ('org'), ('other')` + await client`INSERT INTO workspace(id) VALUES ('workspace')` + }) + afterAll(async () => fixture?.cleanup()) + beforeEach(async () => { + await client`DELETE FROM knowledge_external_group` + }) + + async function group( + id: string, + members: string[], + options: { + provider?: string + tenant?: string + organization?: string + workspace?: string + stale?: boolean + } = {} + ) { + await client`INSERT INTO knowledge_external_group(id, organization_id, workspace_id, provider_id, tenant_id, external_group_id, last_synced_at) + VALUES (${id}, ${options.workspace ? null : (options.organization ?? 'org')}, ${options.workspace ?? null}, + ${options.provider ?? 'confluence'}, ${options.tenant ?? 'cloud'}, ${id}, + ${options.stale ? '2026-09-16T00:00:00Z' : '2026-09-17T12:00:00Z'})` + for (const member of members) { + await client`INSERT INTO knowledge_external_group_member(group_id, subject_token) VALUES (${id}, ${member})` + } + } + + async function parents(seeds = [token('engineering')]) { + const rows = await drizzle(client).execute<{ token: string }>( + parentGroupTokensQuery(seeds, scope, cutoff) + ) + return rows.map((row) => row.token).sort() + } + + it('resolves one native-group hop without traversing arbitrary nested groups or cycles', async () => { + await group('space-readers:1', [token('engineering'), token('cycle')]) + await group('cycle', [token('space-readers:1')]) + await group('space-readers:2', [token('space-readers:1')]) + expect(await parents()).toEqual([token('space-readers:1')]) + expect(await parents([token('space-readers:1')])).toEqual([]) + }) + + it('rejects cross-owner, cross-provider, cross-tenant, and stale membership at every hop', async () => { + await group('space-readers:1', [token('engineering')]) + await group('space-readers:2', [token('engineering')], { organization: 'other' }) + await group('space-readers:3', [token('engineering')], { workspace: 'workspace' }) + await group('space-readers:4', [token('engineering')], { provider: 'jira' }) + await group('space-readers:5', [token('engineering')], { tenant: 'other' }) + await group('space-readers:6', [token('engineering')], { stale: true }) + expect(await parents()).toEqual([token('space-readers:1')]) + expect(await parents([])).toEqual([]) + }) + + it('handles more group seeds than PostgreSQL permits bind parameters', async () => { + await group('space-readers:1', [token('group-69999')]) + const seeds = Array.from({ length: 70_000 }, (_, index) => token(`group-${index}`)) + expect(await parents(seeds)).toEqual([token('space-readers:1')]) + }) +}) diff --git a/apps/sim/lib/knowledge/access/group-membership.test.ts b/apps/sim/lib/knowledge/access/group-membership.test.ts new file mode 100644 index 00000000000..0c9c2ee965f --- /dev/null +++ b/apps/sim/lib/knowledge/access/group-membership.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' +import { + assertExternalGroupTokenCapacity, + MAX_EXTERNAL_GROUP_TOKENS, +} from '@/lib/knowledge/access/group-membership' + +describe('external group token capacity', () => { + it('rejects the overflow sentinel instead of returning a truncated access set', () => { + const tokens = Array.from({ length: MAX_EXTERNAL_GROUP_TOKENS }, () => 'g:confluence:cloud:g') + expect(() => assertExternalGroupTokenCapacity(tokens)).not.toThrow() + expect(() => assertExternalGroupTokenCapacity([...tokens, 'g:confluence:cloud:last'])).toThrow( + 'token capacity' + ) + }) + + it('bounds retained token bytes independently of the number of groups', () => { + const token = `g:confluence:cloud:${'x'.repeat(1024)}` + expect(() => + assertExternalGroupTokenCapacity(Array.from({ length: 16_384 }, () => token)) + ).toThrow('token byte capacity') + }) +}) diff --git a/apps/sim/lib/knowledge/access/group-membership.ts b/apps/sim/lib/knowledge/access/group-membership.ts new file mode 100644 index 00000000000..bf5f8ae9c25 --- /dev/null +++ b/apps/sim/lib/knowledge/access/group-membership.ts @@ -0,0 +1,92 @@ +import { knowledgeExternalGroup, knowledgeExternalGroupMember } from '@sim/db/schema' +import { and, eq, gte, type SQL, sql } from 'drizzle-orm' +import { alias } from 'drizzle-orm/pg-core' +import type { ResourceScope } from '@/lib/core/resource-scope' +import { resourceScopeCondition } from '@/lib/core/resource-scope.server' +import { CONFLUENCE_SPACE_GROUP_PREFIX } from '@/lib/knowledge/access/confluence-space-groups' + +/** A directory exceeding these read-side budgets fails closed instead of truncating access. */ +export const MAX_EXTERNAL_GROUP_TOKENS = 100_000 +const MAX_EXTERNAL_GROUP_TOKEN_BYTES = 16 * 1024 * 1024 + +export function assertExternalGroupTokenCapacity(tokens: readonly string[]): void { + let bytes = 0 + for (const token of tokens) { + bytes += Buffer.byteLength(token, 'utf8') + if (bytes > MAX_EXTERNAL_GROUP_TOKEN_BYTES) { + throw new Error('External group access exceeded its token byte capacity') + } + } + if (tokens.length > MAX_EXTERNAL_GROUP_TOKENS) { + throw new Error('External group access exceeded its token capacity') + } +} + +/** Match a document's audience first, then the same reader directly or through one native group. */ +export function confluenceReaderGroupCondition(input: { + readerSubjectToken: SQL + cloudId: SQL + organizationId: SQL + workspaceId: SQL + freshEnough: SQL + hasToken: (token: SQL) => SQL +}): SQL { + const group = knowledgeExternalGroup + const member = knowledgeExternalGroupMember + const native = alias(group, 'confluence_native_group') + const nativeMember = alias(member, 'confluence_native_member') + const audienceMember = alias(member, 'confluence_audience_member') + const token = sql`('g:' || ${group.providerId} || ':' || ${group.tenantId} || ':' || ${group.externalGroupId})` + return sql` + EXISTS ( + SELECT 1 FROM ${group} + WHERE ${group.providerId} = 'confluence' + AND ${group.tenantId} = ${input.cloudId} + AND ${group.organizationId} IS NOT DISTINCT FROM ${input.organizationId} + AND ${group.workspaceId} IS NOT DISTINCT FROM ${input.workspaceId} + AND ${group.lastSyncedAt} >= ${input.freshEnough} + AND ${input.hasToken(token)} + AND ( + EXISTS ( + SELECT 1 FROM ${member} + WHERE ${member.groupId} = ${group.id} + AND ${member.subjectToken} = ${input.readerSubjectToken} + ) + OR (starts_with(${group.externalGroupId}, ${CONFLUENCE_SPACE_GROUP_PREFIX}) AND EXISTS ( + SELECT 1 FROM ${member} AS ${nativeMember} + INNER JOIN ${group} AS ${native} ON ${native.id} = ${nativeMember.groupId} + INNER JOIN ${member} AS ${audienceMember} ON ${audienceMember.groupId} = ${group.id} + AND ${audienceMember.subjectToken} = 'g:confluence:' || ${native.tenantId} || ':' || ${native.externalGroupId} + WHERE ${nativeMember.subjectToken} = ${input.readerSubjectToken} + AND ${native.providerId} = 'confluence' + AND ${native.tenantId} = ${input.cloudId} + AND ${native.organizationId} IS NOT DISTINCT FROM ${input.organizationId} + AND ${native.workspaceId} IS NOT DISTINCT FROM ${input.workspaceId} + AND ${native.lastSyncedAt} >= ${input.freshEnough} + AND NOT starts_with(${native.externalGroupId}, ${CONFLUENCE_SPACE_GROUP_PREFIX}) + )) + ) + ) + ` +} + +export function parentGroupTokensQuery( + tokens: readonly string[], + scope: ResourceScope, + freshEnough: Date +): SQL { + const group = knowledgeExternalGroup + const member = knowledgeExternalGroupMember + return sql` + SELECT DISTINCT 'g:' || ${group.providerId} || ':' || ${group.tenantId} || ':' || ${group.externalGroupId} AS token + FROM ${group} + INNER JOIN ${member} ON ${eq(member.groupId, group.id)} + WHERE ${and(resourceScopeCondition(group, scope), gte(group.lastSyncedAt, freshEnough))} + AND ${group.providerId} = 'confluence' + AND starts_with(${group.externalGroupId}, ${CONFLUENCE_SPACE_GROUP_PREFIX}) + AND starts_with(${member.subjectToken}, 'g:confluence:' || ${group.tenantId} || ':') + AND NOT starts_with(${member.subjectToken}, 'g:confluence:' || ${group.tenantId} || ':' || ${CONFLUENCE_SPACE_GROUP_PREFIX}) + AND ${member.subjectToken} IN (SELECT jsonb_array_elements_text(${JSON.stringify(tokens)}::text::jsonb)) + LIMIT ${MAX_EXTERNAL_GROUP_TOKENS + 1} + ` +} diff --git a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts index fbd361b4aa8..14168ecf854 100644 --- a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts @@ -250,6 +250,28 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { } const check = (grants?: ConfluenceSiteReadGrant[], userId = 'cf-reader', join = false) => readable([token], 'cf-document', join, undefined, userId, grants) + const audience = 'g:confluence:cloud-1:space-readers:123' + const engineering = 'g:confluence:cloud-1:engineering' + await connection.unsafe(` + INSERT INTO knowledge_external_group(id,organization_id,provider_id,tenant_id,external_group_id,last_synced_at) + VALUES ('cf-native','cf-org','confluence','cloud-1','engineering',now()), + ('cf-space','cf-org','confluence','cloud-1','space-readers:123',now()); + INSERT INTO knowledge_external_group_member(group_id,subject_token) + VALUES ('cf-native','${token}'), ('cf-space','${engineering}'); + UPDATE document SET acl=ARRAY['${audience}'], acl_requirements='[["${engineering}"],["${token}"]]' WHERE id='cf-document'; + `) + const nestedCheck = () => + readable([token, audience, engineering], 'cf-document', true, undefined, 'cf-reader', [grant]) + expect(await nestedCheck()).toBe(true) + await connection`UPDATE knowledge_external_group SET last_synced_at = now() - interval '2 days' WHERE id = 'cf-native'` + expect(await nestedCheck()).toBe(false) + await connection`UPDATE knowledge_external_group SET last_synced_at = now() WHERE id = 'cf-native'` + await connection`UPDATE knowledge_external_group_member SET subject_token = 's:confluence:-:cf-bob' WHERE group_id = 'cf-native'` + expect(await nestedCheck()).toBe(false) + await connection.unsafe(` + DELETE FROM knowledge_external_group WHERE id IN ('cf-native','cf-space'); + UPDATE document SET acl=ARRAY['${token}'], acl_requirements='[["${token}"]]' WHERE id='cf-document'; + `) for (const join of [false, true]) { expect(await check(undefined, 'cf-reader', join)).toBe(false) expect(await check([grant], 'cf-reader', join)).toBe(true) @@ -442,6 +464,16 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { expect(await readable(groups, 'locked')).toBe(false) }) + it('reads safely with an audience larger than the SQL parameter limit', async () => { + const tokens = Array.from( + { length: 70_000 }, + (_, index) => `g:confluence:tenant:space-${index}` + ) + await putDocument('large-directory', [tokens[69_999]], [[tokens[50_000]]]) + expect(await readable(tokens, 'large-directory')).toBe(true) + expect(await readable(tokens.slice(0, 69_999), 'large-directory')).toBe(false) + }) + it('expires mirrored user, group and public grants, including legacy and orphaned source rows', async () => { for (const token of ['u:alice@corp.com', 'g:confluence:tenant:space', 'pub']) { await putDocument(token, [token]) diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index bbce2808f25..d3077014937 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -7,14 +7,13 @@ import { knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, - knowledgeExternalGroup, - knowledgeExternalGroupMember, member, user, } from '@sim/db/schema' import { type SQL, sql } from 'drizzle-orm' import { EXTERNAL_GROUP_STALE_AFTER_MS } from '@/lib/knowledge/access/external-groups' import { SOURCE_ACL_MAX_AGE_MS } from '@/lib/knowledge/access/freshness' +import { confluenceReaderGroupCondition } from '@/lib/knowledge/access/group-membership' import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' import { documentConnectorIsActive } from '@/lib/knowledge/documents/connector-lifecycle' import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' @@ -23,17 +22,15 @@ import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' /** Every Confluence clause must match the same confirmed reader, including that reader's groups. */ function confluenceReaderClause(hasToken: (token: SQL) => SQL): SQL { - return sql`(${hasToken(sql`confluence_read_grant.reader_subject_token`)} OR EXISTS ( - SELECT 1 FROM ${knowledgeExternalGroup} - JOIN ${knowledgeExternalGroupMember} ON ${knowledgeExternalGroupMember.groupId} = ${knowledgeExternalGroup.id} - WHERE ${knowledgeExternalGroupMember.subjectToken} = confluence_read_grant.reader_subject_token - AND ${knowledgeExternalGroup.providerId} = 'confluence' - AND ${knowledgeExternalGroup.tenantId} = confluence_read_grant.cloud_id - AND ${knowledgeExternalGroup.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} - AND ${knowledgeExternalGroup.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} - AND ${knowledgeExternalGroup.lastSyncedAt} >= statement_timestamp() - (${EXTERNAL_GROUP_STALE_AFTER_MS} * interval '1 millisecond') - AND ${hasToken(sql`('g:confluence:' || confluence_read_grant.cloud_id || ':' || ${knowledgeExternalGroup.externalGroupId})`)} - ))` + const groups = confluenceReaderGroupCondition({ + readerSubjectToken: sql`confluence_read_grant.reader_subject_token`, + cloudId: sql`confluence_read_grant.cloud_id`, + organizationId: sql`${knowledgeBase.organizationId}`, + workspaceId: sql`${knowledgeBase.workspaceId}`, + freshEnough: sql`statement_timestamp() - (${EXTERNAL_GROUP_STALE_AFTER_MS} * interval '1 millisecond')`, + hasToken, + }) + return sql`(${hasToken(sql`confluence_read_grant.reader_subject_token`)} OR ${groups})` } /** A cached space grant cannot substitute for the reader's current Confluence site access. */ @@ -168,11 +165,8 @@ function githubInstallationAccessCondition(scope: KnowledgeAccessScope): SQL { /** * The single read-side access predicate: the document's ACL overlaps the - * caller's token set. Tokens are bound as scalars and assembled with - * `ARRAY[...]` because the shared pool runs with `fetch_types: false`, under - * which a JS array bound as one parameter fails at execution (see - * packages/db/db.ts). A literal array also keeps the planner's statistics on - * `acl` usable, which is what lets it choose the GIN index for a selective set. + * caller's token set. Small token sets use literal arrays for GIN planning; + * large directories use one JSON parameter to avoid PostgreSQL's bind limit. * * Additional clauses preserve source intersections. Source-derived grants also * require recent evidence, independent of scheduler health. A drained member @@ -243,11 +237,13 @@ function storedKnowledgeAccessCondition( } /** - * A `text[]` literal assembled from scalar binds, for comparing against an - * ACL column. Every place that compares ACLs builds its array this way, for - * the `fetch_types: false` reason above. + * The pool uses fetch_types: false, so arrays must be constructed from scalar + * parameters. A JSON scalar keeps large sets below PostgreSQL's bind limit. */ export function textArrayLiteral(values: readonly string[]): SQL { + if (values.length > 1000) { + return sql`ARRAY(SELECT jsonb_array_elements_text(${JSON.stringify(values)}::text::jsonb))` + } return sql`ARRAY[${sql.join( values.map((value) => sql`${value}`), sql`, ` diff --git a/apps/sim/lib/knowledge/access/scope.test.ts b/apps/sim/lib/knowledge/access/scope.test.ts index ed6e30a11ea..9635a130a9f 100644 --- a/apps/sim/lib/knowledge/access/scope.test.ts +++ b/apps/sim/lib/knowledge/access/scope.test.ts @@ -53,6 +53,7 @@ vi.mock('@/lib/knowledge/access/connector-permissions', () => ({ loadConnectorPermissionGroupTokens: mockCsvGrants, })) +import { MAX_EXTERNAL_GROUP_TOKENS } from '@/lib/knowledge/access/group-membership' import { createKnowledgeAccessProvider, createUserKnowledgeAccessProvider, @@ -280,7 +281,8 @@ describe('createKnowledgeAccessProvider', () => { const [first, second] = await Promise.all([provider.get(), provider.get()]) expect(first).toBe(second) - expect(dbChainMockFns.select).toHaveBeenCalledTimes(2) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(1) + expect(dbChainMockFns.selectDistinct).toHaveBeenCalledTimes(1) }) it('scopes live-source discovery to sources a held reader credential can prove', async () => { @@ -354,6 +356,34 @@ describe('tokens mirrored from a source directory', () => { queueTableRows(schemaMock.knowledgeExternalGroupMember, rows) } + it('fails closed on the direct-group overflow sentinel before discarding malformed tokens', async () => { + queueSubjects([ + { providerId: 'confluence', providerTenantId: null, providerSubjectId: 'reader' }, + ]) + queueGroups( + Array.from({ length: MAX_EXTERNAL_GROUP_TOKENS + 1 }, () => ({ + providerId: 'invalid:provider', + tenantId: 'cloud', + externalGroupId: 'group', + })) + ) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).rejects.toThrow('token capacity') + expect(dbChainMockFns.execute).not.toHaveBeenCalled() + }) + + it('fails closed on the audience overflow sentinel before merging duplicate tokens', async () => { + queueSubjects([ + { providerId: 'confluence', providerTenantId: null, providerSubjectId: 'reader' }, + ]) + queueGroups([{ providerId: 'confluence', tenantId: 'cloud', externalGroupId: 'group' }]) + dbChainMockFns.execute.mockResolvedValueOnce( + Array.from({ length: MAX_EXTERNAL_GROUP_TOKENS + 1 }, () => ({ + token: 'g:confluence:cloud:group', + })) + ) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).rejects.toThrow('token capacity') + }) + it('gives a person their own address and every group it belongs to', async () => { queueSubjects([ { diff --git a/apps/sim/lib/knowledge/access/scope.ts b/apps/sim/lib/knowledge/access/scope.ts index 2a534bc1760..b5120b36458 100644 --- a/apps/sim/lib/knowledge/access/scope.ts +++ b/apps/sim/lib/knowledge/access/scope.ts @@ -35,6 +35,11 @@ import { type GitHubReaderCredential, resolveGitHubInstallationReadGrants, } from '@/lib/knowledge/access/github-installation' +import { + assertExternalGroupTokenCapacity, + MAX_EXTERNAL_GROUP_TOKENS, + parentGroupTokensQuery, +} from '@/lib/knowledge/access/group-membership' import { confluenceSiteSourceCondition, githubInstallationSourceCondition, @@ -105,7 +110,7 @@ async function loadExternalGroupTokens( */ const freshEnough = new Date(Date.now() - EXTERNAL_GROUP_STALE_AFTER_MS) const rows = await db - .select({ + .selectDistinct({ providerId: knowledgeExternalGroup.providerId, tenantId: knowledgeExternalGroup.tenantId, externalGroupId: knowledgeExternalGroup.externalGroupId, @@ -122,6 +127,10 @@ async function loadExternalGroupTokens( gte(knowledgeExternalGroup.lastSyncedAt, freshEnough) ) ) + .limit(MAX_EXTERNAL_GROUP_TOKENS + 1) + if (rows.length > MAX_EXTERNAL_GROUP_TOKENS) { + throw new Error('External group access exceeded its token capacity') + } const tokens: string[] = [] for (const row of rows) { @@ -132,7 +141,19 @@ async function loadExternalGroupTokens( }) if (token) tokens.push(token) } - return tokens + assertExternalGroupTokenCapacity(tokens) + if (tokens.length > 0) { + const parents = await db.execute<{ token: string }>( + parentGroupTokensQuery(tokens, scope, freshEnough) + ) + if (parents.length > MAX_EXTERNAL_GROUP_TOKENS) { + throw new Error('External group access exceeded its token capacity') + } + for (const parent of parents) tokens.push(parent.token) + } + const uniqueTokens = sortAccessTokens(tokens) + assertExternalGroupTokenCapacity(uniqueTokens) + return uniqueTokens } export interface KnowledgeAccessScopeContext { diff --git a/apps/sim/lib/knowledge/access/tokens.test.ts b/apps/sim/lib/knowledge/access/tokens.test.ts index 13c7fbb2d0a..f9ca916d664 100644 --- a/apps/sim/lib/knowledge/access/tokens.test.ts +++ b/apps/sim/lib/knowledge/access/tokens.test.ts @@ -6,6 +6,7 @@ import { ACCESS_TOKEN_PATTERN, groupToken, isAccessToken, + isDirectoryMemberToken, isIdentityToken, MAX_ACL_TOKENS, sortAccessTokens, @@ -178,3 +179,44 @@ describe('directory identity tokens', () => { expect(isIdentityToken(token)).toBe(false) }) }) + +describe('nested directory memberships', () => { + const directory = { + providerId: 'confluence', + tenantId: 'cloud', + externalGroupId: 'space-readers:123', + } + it.each(['s:confluence:-:account', 'g:confluence:cloud:engineering'])( + 'accepts %s within the directory', + (value) => { + expect(isDirectoryMemberToken(value, directory)).toBe(true) + } + ) + it.each([ + 'g:jira:cloud:engineering', + 'g:confluence:other:engineering', + 'g:confluence:cloud:Engineering', + 'g:confluence:cloud: engineering ', + 'g:confluence:cloud:space-readers:456', + 'ws', + 'link', + 'pub', + ])('rejects %s', (value) => { + expect(isDirectoryMemberToken(value, directory)).toBe(false) + }) + + it('rejects nesting in native groups and other providers', () => { + expect( + isDirectoryMemberToken('g:confluence:cloud:engineering', { + ...directory, + externalGroupId: 'native', + }) + ).toBe(false) + expect( + isDirectoryMemberToken('g:google-drive:cloud:engineering', { + ...directory, + providerId: 'google-drive', + }) + ).toBe(false) + }) +}) diff --git a/apps/sim/lib/knowledge/access/tokens.ts b/apps/sim/lib/knowledge/access/tokens.ts index 5677283a584..59fc8e89f4c 100644 --- a/apps/sim/lib/knowledge/access/tokens.ts +++ b/apps/sim/lib/knowledge/access/tokens.ts @@ -1,4 +1,8 @@ import { normalizeEmail } from '@sim/utils/string' +import { + CONFLUENCE_SPACE_GROUP_PREFIX, + isConfluenceSpaceGroupId, +} from '@/lib/knowledge/access/confluence-space-groups' import { type MirroredDocumentAcl, WORKSPACE_ACCESS_TOKEN } from '@/lib/knowledge/access/types' /** @@ -86,13 +90,30 @@ export function isAccessToken(value: string): boolean { return ACCESS_TOKEN_PATTERN.test(value) } -/** Directory members are people; document audiences and groups cannot become member identities. */ +/** Identity seeds represent people; document audiences and groups are not reader identities. */ export function isIdentityToken(value: string): boolean { if (!isAccessToken(value)) return false if (value.startsWith('s:')) return true return value.startsWith('u:') && userToken(value.slice(2)) === value } +/** A Confluence space audience may reference native groups; native memberships remain identities. */ +export function isDirectoryMemberToken( + value: string, + directory: { providerId: string; tenantId: string; externalGroupId: string } +): boolean { + if (isIdentityToken(value)) return true + if (directory.providerId !== 'confluence' || !isConfluenceSpaceGroupId(directory.externalGroupId)) + return false + const prefix = `g:${directory.providerId}:${directory.tenantId || NO_TENANT_SEGMENT}:` + return ( + isAccessToken(value) && + value.startsWith(prefix) && + !value.slice(prefix.length).startsWith(CONFLUENCE_SPACE_GROUP_PREFIX) && + value === groupToken({ ...directory, groupId: value.slice(prefix.length) }) + ) +} + export interface SubjectCredential { providerId: string | null providerTenantId: string | null diff --git a/apps/sim/lib/knowledge/application/personal-source-setup.test.ts b/apps/sim/lib/knowledge/application/personal-source-setup.test.ts index 9ae1a409a4d..b40ff48ee5a 100644 --- a/apps/sim/lib/knowledge/application/personal-source-setup.test.ts +++ b/apps/sim/lib/knowledge/application/personal-source-setup.test.ts @@ -90,6 +90,46 @@ const runConnect = (changes = {}) => personalSourceSetup.execute({ principal, input: { ...connect, keys: ['PROJECT'], ...changes } }) describe('personal source setup', () => { + it.each(['confluence', 'jira'] as const)( + 'rejects mixed All and explicit %s keys before discovery or saving', + async (connectorType) => { + await expect(runConnect({ connectorType, keys: ['*', 'ENG'] })).rejects.toThrow( + 'Use "*" by itself for All, or remove it to select individual items.' + ) + expect(mocks.selector).not.toHaveBeenCalled() + expect(mocks.configure).not.toHaveBeenCalled() + } + ) + + it.each(['confluence', 'jira'] as const)( + 'authorizes %s All without freezing the available keys', + async (connectorType) => { + mocks.selector.mockResolvedValue({ kind: 'list', items: [], nextCursor: 'more' }) + await expect(runConnect({ connectorType, keys: ['*'] })).resolves.toMatchObject({ + kind: 'connected', + }) + expect(mocks.ownAccount).toHaveBeenCalled() + expect(mocks.selector).toHaveBeenCalledTimes(1) + expect(mocks.configure).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: expect.objectContaining({ + sourceConfig: { + domain: credential.domain, + [connectorType === 'confluence' ? 'spaceKey' : 'projectKey']: '*', + }, + }), + }) + ) + } + ) + + it('does not let All bypass a failed site or credential check', async () => { + mocks.selector.mockRejectedValue(new Error('Site unavailable')) + await expect(runConnect({ keys: ['*'] })).rejects.toThrow('Site unavailable') + expect(mocks.configure).not.toHaveBeenCalled() + }) + afterEach(() => { vi.restoreAllMocks() }) diff --git a/apps/sim/lib/knowledge/application/personal-source-setup.ts b/apps/sim/lib/knowledge/application/personal-source-setup.ts index 407ad85b918..e38870f5103 100644 --- a/apps/sim/lib/knowledge/application/personal-source-setup.ts +++ b/apps/sim/lib/knowledge/application/personal-source-setup.ts @@ -28,6 +28,7 @@ import { MAX_SELECTOR_PAGES } from '@/lib/selectors/limits' import type { SelectorExecutionResult, SelectorRequest } from '@/lib/selectors/types' import { MAX_PERSONAL_SOURCE_SETUP_KEYS } from '@/lib/sim-search/personal-source-setup' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import { getSourceSelectionError, isAllSourceItems } from '@/connectors/selection' const logger = createLogger('PersonalSourceSetup') const VALIDATION_PHASE_TIMEOUT_MS = 30_000 @@ -177,6 +178,8 @@ export const personalSourceSetup = defineAuthorizedKnowledgeUseCase({ throw new OrchestrationError('validation', 'Select between 1 and 1,000 projects or spaces') } const keys = [...new Set(input.keys.map((key) => key.trim()))] + const selectionError = getSourceSelectionError(keys) + if (selectionError) throw new OrchestrationError('validation', selectionError) const remaining = new Set(keys) const cursors = new Set() const timeout = AbortSignal.timeout(VALIDATION_PHASE_TIMEOUT_MS) @@ -201,6 +204,10 @@ export const personalSourceSetup = defineAuthorizedKnowledgeUseCase({ throw error } if (result.kind !== 'list') throw new Error('Source discovery returned an unexpected result') + if (isAllSourceItems(keys)) { + remaining.clear() + break + } for (const option of result.items) remaining.delete(option.id) if (remaining.size === 0) break if (!result.nextCursor || result.truncated || cursors.has(result.nextCursor)) break diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts index 85ee6179525..82082479983 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts @@ -93,6 +93,19 @@ describe('syncExternalDirectoryGroups', () => { expect(dir.listGroups).toHaveBeenCalledOnce() }) + it('rejects group references in a native directory membership', async () => { + const dir = directory({ + listGroupMembers: vi.fn(async (group) => ({ + group, + memberTokens: ['g:google-drive:corp.com:engineering'], + complete: true, + })), + }) + await expect( + syncExternalDirectoryGroups({ workspaceId: 'ws-1', directory: dir }) + ).rejects.toThrow('invalid identity token') + }) + it('replaces membership only from a complete enumeration, keeping the rest last-known-good', async () => { queueTableRows(schemaMock.knowledgeExternalGroup, []) const dir = directory({ diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.ts index be56c0abe40..a5b24273f58 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.ts @@ -20,8 +20,9 @@ import { import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import type { DbTransaction } from '@/lib/db/types' import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' +import { CONFLUENCE_SPACE_GROUP_PREFIX } from '@/lib/knowledge/access/confluence-space-groups' import { EXTERNAL_GROUP_SYNC_INTERVAL_MS } from '@/lib/knowledge/access/external-groups' -import { canonicalGroupId, isIdentityToken } from '@/lib/knowledge/access/tokens' +import { canonicalGroupId, isDirectoryMemberToken } from '@/lib/knowledge/access/tokens' import { mirrorsSourceAcls } from '@/lib/knowledge/connectors/access-modes' import { resolveConnectorAccessToken, @@ -37,6 +38,7 @@ import { ConnectorDirectoryGroupAccessError, } from '@/connectors/source-error' import type { + ConnectorAclGroupMembership, ConnectorConfig, ConnectorDirectory, ConnectorDirectoryGroup, @@ -228,6 +230,7 @@ export async function syncExternalDirectoryGroups(input: { const groupId = await withDirectoryLease(lease, (tx) => upsertGroup({ ...owner, providerId, tenantId, group }, tx) ) + if (!groupId) throw new Error('Directory group could not be persisted') let membership: ConnectorDirectoryMembership try { membership = await directory.listGroupMembers(group) @@ -253,7 +256,12 @@ export async function syncExternalDirectoryGroups(input: { continue } await withDirectoryLease(lease, (tx) => - replaceGroupMembers(groupId, membership.memberTokens, tx) + replaceGroupMembers( + groupId, + membership.memberTokens, + { ...lease, externalGroupId: group.id }, + tx + ) ) refreshed += 1 } @@ -293,9 +301,10 @@ export async function syncExternalDirectoryGroups(input: { async function upsertGroup( input: DirectoryIdentity & { group: ConnectorDirectoryGroup }, - tx: DbTransaction -): Promise { - const { workspaceId, providerId, tenantId, group } = input + tx: DbTransaction, + observedAt?: Date +): Promise { + const { providerId, tenantId, group } = input const [row] = await tx .insert(knowledgeExternalGroup) .values({ @@ -315,18 +324,44 @@ async function upsertGroup( knowledgeExternalGroup.externalGroupId, ], set: { updatedAt: new Date() }, + ...(observedAt + ? { + setWhere: or( + isNull(knowledgeExternalGroup.lastSyncedAt), + lt(knowledgeExternalGroup.lastSyncedAt, observedAt) + ), + } + : {}), }) .returning({ id: knowledgeExternalGroup.id }) - return row.id + return row?.id +} + +/** The caller owns the transaction and fences it with its directory or content-sync lease. */ +export async function persistExternalGroupMembership( + input: DirectoryIdentity & ConnectorAclGroupMembership & { observedAt: Date }, + tx: DbTransaction +): Promise { + const groupId = await upsertGroup(input, tx, input.observedAt) + if (!groupId) return + await replaceGroupMembers( + groupId, + input.memberTokens, + { ...input, externalGroupId: input.group.id }, + tx, + input.observedAt + ) } -/** Membership replacement and its freshness watermark commit together under the directory lease. */ +/** Membership replacement and its freshness watermark commit together. */ async function replaceGroupMembers( groupId: string, memberTokens: string[], - tx: DbTransaction + directory: DirectoryIdentity & { externalGroupId: string }, + tx: DbTransaction, + observedAt?: Date ): Promise { - if (memberTokens.some((token) => !isIdentityToken(token))) { + if (memberTokens.some((token) => !isDirectoryMemberToken(token, directory))) { throw new Error('Directory membership contains an invalid identity token') } await tx @@ -339,7 +374,7 @@ async function replaceGroupMembers( } await tx .update(knowledgeExternalGroup) - .set({ lastSyncedAt: sql`clock_timestamp()`, updatedAt: sql`clock_timestamp()` }) + .set({ lastSyncedAt: observedAt ?? sql`clock_timestamp()`, updatedAt: sql`clock_timestamp()` }) .where(eq(knowledgeExternalGroup.id, groupId)) } @@ -358,6 +393,11 @@ async function pruneRemovedGroups(lease: DirectoryLease, keep: readonly string[] eq(knowledgeExternalGroup.tenantId, lease.tenantId), ...(keep.length > 0 ? [notInArray(knowledgeExternalGroup.externalGroupId, [...keep])] + : []), + ...(lease.providerId === 'confluence' + ? [ + sql`NOT starts_with(${knowledgeExternalGroup.externalGroupId}, ${CONFLUENCE_SPACE_GROUP_PREFIX})`, + ] : []) ) ) diff --git a/apps/sim/lib/knowledge/connectors/member-access.test.ts b/apps/sim/lib/knowledge/connectors/member-access.test.ts index a56745e6fe0..79821685197 100644 --- a/apps/sim/lib/knowledge/connectors/member-access.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-access.test.ts @@ -74,6 +74,7 @@ import { ResourcePolicyNotFoundError, ResourcePolicyRevisionConflictError, } from '@/lib/resource-policies/repository' +import { confluenceConnectorMeta } from '@/connectors/confluence/meta' const GROUP_ID = 'group-1' const BINDING = { @@ -518,6 +519,34 @@ describe('knowledge connector member access', () => { ).toEqual({ ok: true, option: driveOption }) }) + it('keeps existing Confluence page credentials eligible without attachment access', () => { + const confluenceOption = { + ...driveOption, + id: 'option-confluence', + provider: 'confluence', + label: 'Confluence', + authorizationAppId: 'atlassian:app', + requiredScopes: [ + 'read:confluence-content.all', + 'read:page:confluence', + 'read:blogpost:confluence', + 'read:space:confluence', + 'read:label:confluence', + 'search:confluence', + 'offline_access', + ], + } + + expect( + validateKnowledgeConnectorMembersBinding({ + connectorMeta: confluenceConnectorMeta, + group: { status: 'active', options: [confluenceOption] }, + credentialGroupOptionId: confluenceOption.id, + sourceConfig: { domain: 'example.atlassian.net', spaceKey: ['ENG'], maxPages: '' }, + }) + ).toEqual({ ok: true, option: confluenceOption }) + }) + describe('a Slack option, whose members authorize through the workspace custom app', () => { const slackMeta = { name: 'Slack', diff --git a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts index 5ab9a1ee5d1..6539a0dfe88 100644 --- a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts @@ -128,6 +128,7 @@ beforeEach(() => { version: { number: sourceVersion }, } if (url.pathname.endsWith('/spaces/space/pages')) return Response.json({ results: [page] }) + if (url.pathname.endsWith('/pages/page/attachments')) return Response.json({ results: [] }) if (url.pathname.endsWith('/pages/page')) { return Response.json({ ...page, @@ -421,6 +422,46 @@ describe('content pass checkpoint intent', () => { expect(result.docsUpdated).toBe(0) }) + it.each([403, 401])( + 'indexes healthy Confluence pages while preserving attachments after access failure %s', + async (status) => { + sourceBody = { value: '

Current content

' } + const healthy = vi.mocked(fetch).getMockImplementation()! + vi.mocked(fetch).mockImplementation(async (input, init) => + String(input).includes('/pages/page/attachments') + ? status === 401 + ? Response.json( + { code: 401, message: 'Unauthorized; scope does not match' }, + { status } + ) + : new Response(null, { status }) + : healthy(input, init) + ) + const { pass, result } = await runPass({ access: 'admin' }) + expect(pass.complete).toBe(true) + expect(pass.checkpoint).toMatchObject({ + unsafe: true, + listingFailures: { + count: 1, + samples: [ + { + scope: 'page', + operation: 'confluence.attachments.list', + status, + reasons: [status === 401 ? 'attachment_scope_mismatch' : 'attachment_access_denied'], + }, + ], + }, + }) + expect(pass.holdNotice).toContain('unlisted documents were kept') + expect(result.docsAdded).toBe(1) + expect(result.docsDeleted).toBe(0) + expect(mocks.dispatch).toHaveBeenCalledOnce() + expect(mocks.hardDelete).not.toHaveBeenCalled() + expect(dbChainMockFns.set.mock.calls.some(([value]) => value.deletedAt != null)).toBe(false) + } + ) + it('does not reconcile deletions after a user listing failed, even without the unsafe marker', async () => { sourceBody = { value: '

Current content

' } const checkpoint = { diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index c39b5524d43..6f2633b6168 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -18,6 +18,7 @@ import { import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { EMPTY_ACL } from '@/lib/knowledge/access/tokens' +import type { MirroredDocumentAcl } from '@/lib/knowledge/access/types' import { CONTENT_ENGINE_ACCESS_MODES, type ContentEngineAccessMode, @@ -36,6 +37,7 @@ import { type DirectoryRefreshResult, directorySyncNotice, hasDirectorySyncNotice, + persistExternalGroupMembership, refreshMirroredDirectory, } from '@/lib/knowledge/connectors/external-group-sync' import { listingFingerprint } from '@/lib/knowledge/connectors/listing-checkpoint' @@ -116,6 +118,7 @@ export { */ async function applySourceMirroredAcls(input: { connectorId: string + kbOwner: KnowledgeBaseOwner connectorConfig: ConnectorConfig sourceConfig: Record syncContext: Record @@ -135,15 +138,36 @@ async function applySourceMirroredAcls(input: { * place until the next run. */ const unanswered = unansweredByListing(externalDocs) - const fetched = - unanswered.length > 0 && connectorConfig.getDocumentAcls - ? await connectorConfig.getDocumentAcls( - input.accessToken, - input.sourceConfig, - unanswered, - input.syncContext - ) - : {} + let fetched: Record = {} + if (unanswered.length > 0 && connectorConfig.getDocumentAcls) { + /** Audience freshness belongs to this observation, including when an old crawl resumes. */ + const [clock] = await db.execute<{ startedAt: string }>( + sql`SELECT statement_timestamp()::text AS "startedAt"` + ) + const observedAt = new Date(clock?.startedAt ?? '') + if (!Number.isFinite(observedAt.getTime())) + throw new Error('Could not read the sync database clock') + fetched = await connectorConfig.getDocumentAcls( + input.accessToken, + input.sourceConfig, + unanswered, + input.syncContext, + { + persistGroupMembership: (membership) => + db.transaction(async (tx) => { + if (input.lease) await assertSyncLeaseHeldInTx(tx, connectorId, input.lease) + await persistExternalGroupMembership( + { + ...resourceScopeFields(resourceScopeFromOwner(input.kbOwner)), + ...membership, + observedAt, + }, + tx + ) + }), + } + ) + } const { acls, unattributed, unresolvedExternalIds } = mergeMirroredAcls(externalDocs, fetched) const evidence = { unresolvedExternalIds, generationStartedAt: input.generationStartedAt } const listed = acls.size @@ -1158,6 +1182,7 @@ export async function executeSync( await directoryRefreshed return applySourceMirroredAcls({ connectorId, + kbOwner, connectorConfig, sourceConfig, syncContext, diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index 2a08a2bc246..e9606037f74 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -101,8 +101,23 @@ vi.mock('@/connectors/registry.server', () => ({ }, notion: { auth: { mode: 'apiKey', optional: true }, + configFields: [], validateConfig: vi.fn().mockResolvedValue({ valid: true }), }, + jira: { + name: 'Jira', + auth: { mode: 'oauth', provider: 'jira' }, + permissionScopedListing: { capFieldIds: [] }, + configFields: [ + { id: 'projectSelector', canonicalParamId: 'projectKey', selectAllValue: '*' }, + ], + }, + confluence: { + name: 'Confluence', + auth: { mode: 'oauth', provider: 'confluence' }, + permissionScopedListing: { capFieldIds: [] }, + configFields: [{ id: 'spaceSelector', canonicalParamId: 'spaceKey', selectAllValue: '*' }], + }, google_drive: { name: 'Google Drive', auth: { mode: 'oauth', provider: 'google-drive' }, @@ -155,6 +170,30 @@ describe('performCreateKnowledgeConnector', () => { resolveAccessToken: vi.fn(), } + it.each([ + { connectorType: 'jira', field: 'projectKey' }, + { connectorType: 'confluence', field: 'spaceKey' }, + ])( + 'rejects mixed All keys for credentialless $connectorType members', + async ({ connectorType, field }) => { + const outcome = await performCreateKnowledgeConnector({ + ...createParams, + connectorType, + sourceConfig: { domain: 'example.atlassian.net', [field]: ['*', 'ENG'] }, + accessMode: 'members', + membersBinding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + }) + expect(outcome).toMatchObject({ + success: false, + errorCode: 'validation', + error: 'Use "*" by itself for All, or remove it to select individual items.', + }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + expect(mockGrant).not.toHaveBeenCalled() + expect(createParams.resolveAccessToken).not.toHaveBeenCalled() + } + ) + it('validates and encrypts a GitHub PAT without resolving an OAuth account or returning the secret', async () => { dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'kb-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([ @@ -481,6 +520,41 @@ describe('performUpdateKnowledgeConnector', () => { afterAll(resetDbChainMock) + it.each([ + { connectorType: 'jira', field: 'projectKey' }, + { connectorType: 'confluence', field: 'spaceKey' }, + ])( + 'rejects mixed All keys when editing credentialless $connectorType members', + async ({ connectorType, field }) => { + queueTableRows(schemaMock.knowledgeConnector, [ + { + id: 'conn-1', + connectorType, + accessMode: 'members', + status: 'active', + memberSyncStatus: 'idle', + credentialId: null, + }, + ]) + const validateSourceConfig = vi.fn() + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { sourceConfig: { domain: 'example.atlassian.net', [field]: '*, ENG' } }, + resolveBillingAttribution, + validateSourceConfig, + }) + expect(outcome).toMatchObject({ + success: false, + errorCode: 'validation', + error: 'Use "*" by itself for All, or remove it to select individual items.', + }) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(validateSourceConfig).not.toHaveBeenCalled() + } + ) + it('rejects an update that names nothing before reading the connector', async () => { const outcome = await performUpdateKnowledgeConnector({ ...ACTOR, diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index f3ee6da4858..209d500ea68 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -64,6 +64,7 @@ import { createTagDefinition } from '@/lib/knowledge/tags/service' import { captureServerEvent } from '@/lib/posthog/server' import { searchSourceIdentity } from '@/lib/sim-search/source-identity' import { getConnectorApiKeyConfig } from '@/connectors/auth' +import { findSourceSelectionError } from '@/connectors/selection' import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' const logger = createLogger('KnowledgeConnectorOrchestration') @@ -259,6 +260,9 @@ export async function performCreateKnowledgeConnector( return fail(`Unknown connector type: ${connectorType}`, 'validation') } + const selectionError = findSourceSelectionError(connectorConfig, sourceConfig) + if (selectionError) return fail(selectionError, 'validation') + try { await assertLiveSyncAllowed(resourceScopeFromOwner(kb), syncIntervalMinutes) } catch (error) { @@ -825,11 +829,13 @@ export async function performUpdateKnowledgeConnector( let nextSourceConfig = params.prepareSourceConfig ? await params.prepareSourceConfig(existing, updates.sourceConfig) : updates.sourceConfig - if (aclIsDerived(accessMode)) { - /** A derived-ACL mode has no listing cap; a save may refuse one, never store one. */ - const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') - const connectorConfig = CONNECTOR_REGISTRY[existing.connectorType] - if (connectorConfig) { + const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') + const connectorConfig = CONNECTOR_REGISTRY[existing.connectorType] + if (connectorConfig) { + const selectionError = findSourceSelectionError(connectorConfig, nextSourceConfig) + if (selectionError) return fail(selectionError, 'validation') + if (aclIsDerived(accessMode)) { + /** A derived-ACL mode has no listing cap; a save may refuse one, never store one. */ const capViolation = findListingCapViolation(connectorConfig, nextSourceConfig) if (capViolation) return fail(capViolation, 'validation') nextSourceConfig = stripListingCapFields(connectorConfig, nextSourceConfig) diff --git a/apps/sim/lib/sim-search/personal-source-setup.ts b/apps/sim/lib/sim-search/personal-source-setup.ts index 8afb1049c9f..7155fd26ed9 100644 --- a/apps/sim/lib/sim-search/personal-source-setup.ts +++ b/apps/sim/lib/sim-search/personal-source-setup.ts @@ -1,2 +1,2 @@ -/** Personal Atlassian setup stores an explicit, bounded snapshot of selected project or space keys. */ +/** Personal Atlassian setup bounds explicitly selected project or space keys. */ export const MAX_PERSONAL_SOURCE_SETUP_KEYS = 1000 diff --git a/apps/sim/lib/sim-search/source-identity.test.ts b/apps/sim/lib/sim-search/source-identity.test.ts index 67208256a0d..175202a5641 100644 --- a/apps/sim/lib/sim-search/source-identity.test.ts +++ b/apps/sim/lib/sim-search/source-identity.test.ts @@ -14,6 +14,18 @@ import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' describe('Search source identity', () => { + it('describes dynamic All without exposing the persisted marker', () => { + expect( + describeSearchSource(confluenceConnectorMeta, { + domain: 'example.atlassian.net', + spaceKey: ['*'], + }) + ).toBe('example.atlassian.net · All') + expect(searchSourceIdentity(confluenceConnectorMeta, { spaceKey: ['*'] })).not.toBe( + searchSourceIdentity(confluenceConnectorMeta, { spaceKey: ['ENG', 'HR'] }) + ) + }) + it('normalizes multi-value settings and ignores runtime mappings and cleared caps', () => { expect( searchSourceIdentity(confluenceConnectorMeta, { diff --git a/apps/sim/lib/sim-search/source-identity.ts b/apps/sim/lib/sim-search/source-identity.ts index 5f6875948e8..2cba485f134 100644 --- a/apps/sim/lib/sim-search/source-identity.ts +++ b/apps/sim/lib/sim-search/source-identity.ts @@ -1,5 +1,6 @@ import { isPlainRecord } from '@sim/utils/object' import { truncate } from '@sim/utils/string' +import { isAllSourceItems } from '@/connectors/selection' import type { ConnectorMeta } from '@/connectors/types' import { parseMultiValue } from '@/connectors/utils' @@ -175,6 +176,7 @@ export function describeSearchSource( .flatMap((id) => { if (!SOURCE_ADDRESS_FIELDS.has(id) || caps.has(id)) return [] const value = sourceConfig[id] + if ((id === 'spaceKey' || id === 'projectKey') && isAllSourceItems(value)) return ['All'] if (labels[id]) return labels[id].map((option) => option.label) const values = parseMultiValue(value) const hasOpaqueId = values.some((item) => diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 0be0f42f8d2..48db7dfb4b9 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -6037,8 +6037,8 @@ export const knowledgeExternalGroup = pgTable( /** * External group membership keyed by canonical identity tokens: verified * addresses (`u:`) or provider account identities (`s:`). Provider identities - * preserve permissions when a directory hides email addresses. Nested groups - * are flattened by directory sync, without requiring members to have Sim accounts. + * preserve permissions when a directory hides email addresses. Confluence space + * audiences may also reference native groups, whose members remain identities. */ export const knowledgeExternalGroupMember = pgTable( 'knowledge_external_group_member',