feat(atlassian): resolve dynamic scopes and index Confluence attachments (#7935)

* feat(atlassian): resolve dynamic scopes and index Confluence attachments

* fix(atlassian): reject mixed All scopes and run permission tests in CI

* fix(ci): isolate Confluence migration test database

Author: waleedlatif1

.github/workflows/test-build.yml

@@ -245,6 +245,26 @@ jobs:
           lib/knowledge/__integration__/connector-upload.integration.ts
           lib/uploads/contexts/organization-logo/application.integration.ts
 
+      - name: Verify Confluence identity and directory sync in PostgreSQL
+        working-directory: apps/sim
+        env:
+          KNOWLEDGE_ACL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
+        run: >-
+          bunx vitest run --mode integration
+          lib/knowledge/__integration__/confluence-identity.integration.ts
+          lib/knowledge/__integration__/directory-sync.integration.ts
+
+      - name: Verify Confluence audience migrations and permission queries in PostgreSQL
+        working-directory: apps/sim
+        env:
+          KNOWLEDGE_ACL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_acl_test
+        run: |
+          bun -e 'import postgres from "postgres"; const sql = postgres(process.env.DATABASE_URL); await sql`CREATE DATABASE sim_acl_test`; await sql.end()'
+          bunx vitest run --mode integration lib/knowledge/access/group-membership.integration.ts
+          bunx vitest run \
+            lib/knowledge/access/predicate.postgres.test.ts \
+            lib/knowledge/connectors/external-directory.postgres.test.ts
+
   test-build:
     name: Lint and Test
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}

apps/docs/content/docs/knowledgebase/connectors.mdx

@@ -78,7 +78,7 @@ Each connector has source-specific fields that control what gets synced. Example
 
 - **Notion** — sync an entire workspace, a specific database, or a single page tree
 - **GitHub** — specify a repository, branch, and optional file extension filter
-- **Confluence** — enter your Atlassian domain and optionally filter by space key or content type
+- **Confluence** — enter your Atlassian domain and choose spaces, or **All** for all spaces accessible at each sync. Optionally filter by content type or label. PDF and Word (`.docx`, Word 97–2003 `.doc`) attachments on matching pages and blog posts are included as separate documents.
 - **Azure DevOps** — choose what to sync (wiki pages, work items, repository files, or all), with optional work item type/state filters, a custom WIQL query, and repository/branch/path filters
 - **Amazon S3** — point at a bucket with an optional key prefix and a customizable file extension allowlist; S3-compatible stores (Cloudflare R2, MinIO) are supported via a custom endpoint
 - **YouTube** — sync a channel (by `@handle` or ID) or playlist, with an optional published-after date filter and the option to exclude Shorts
@@ -88,6 +88,8 @@ Each connector has source-specific fields that control what gets synced. Example
 
 Configuration is validated on save — if a repository doesn't exist or a domain is unreachable, you'll see an error immediately.
 
+Confluence attachment indexing requires `read:attachment:confluence`. For a service account, include it when creating the scoped API token; see the [Confluence scope list](/search/confluence#using-a-service-account). Attachments are checked even when the parent page has not changed. Files over 100 MB appear as skipped; convert Word 6/95 files to `.docx` before attaching them.
+
 </Step>
 <Step>
 

apps/docs/content/docs/search/confluence.mdx

@@ -7,7 +7,7 @@ import { Callout } from 'fumadocs-ui/components/callout'
 import { Step, Steps } from 'fumadocs-ui/components/steps'
 import { Image } from '@/components/ui/image'
 
-Search pages and blog posts from selected Confluence Cloud spaces. A Sim organization admin enables Confluence; **each teammate connects their own account**.
+Search pages, blog posts, and their PDF and Word attachments from selected Confluence Cloud spaces. A Sim organization admin enables Confluence; **each teammate connects their own account**.
 
 | Method | How it works |
 | --- | --- |
@@ -38,7 +38,9 @@ Open **Settings → Sources → Add source** and select **Confluence**. This ope
 
 ### Choose the account and spaces
 
-Under **Service account**, select a service account or [add one](#using-a-service-account). Enter the same **Confluence site** as the credential, then choose **Spaces**. **All** in the dropdown selects every space the account can currently browse; newly created spaces are not added automatically. Clear the picker search before selecting all.
+Under **Service account**, select a service account or [add one](#using-a-service-account). Enter the same **Confluence site** as the credential, then choose **Spaces**. **All** in the dropdown includes every space the syncing account can access at each sync, including newly accessible spaces. Clear the picker search before selecting all.
+
+If you selected all spaces before this behavior was introduced, reselect **All** and save. Previously saved selections remain a fixed list of spaces.
 
 To enter comma-separated keys such as `ENG, PRODUCT`, use the switch beside **Spaces**. Switching between the picker and manual entry keeps your selection.
 
@@ -63,7 +65,7 @@ After an admin configures Confluence, open **Integrations** and select **Connect
 If Confluence is allowed but no source exists, select **Connect** beside Confluence. To add another site later, open the Confluence row’s actions menu (**…**) and select **Add Confluence site**:
 
 1. Open **Your account** and select a saved account or **Connect Confluence account**. Authorize using the Atlassian email matching your verified Sim email.
-2. Enter the hostname under **Atlassian site**, then choose **Spaces**. Use **All** in the dropdown for the complete current list, or the arrows beside **Spaces** to enter comma-separated keys. You can select up to 1,000 spaces in this form.
+2. Enter the hostname under **Atlassian site**, then choose **Spaces**. Use **All** in the dropdown for all spaces accessible at each sync, or the arrows beside **Spaces** to enter comma-separated keys. You can select up to 1,000 individual spaces in this form.
 3. Select **Connect & Sync**. Sim saves the selected scope and starts indexing with your account.
 
 <Image className="mx-auto h-auto w-full max-w-md"
@@ -87,6 +89,7 @@ Use a **scoped API token** from an Atlassian service account:
 read:confluence-content.all
 read:page:confluence
 read:blogpost:confluence
+read:attachment:confluence
 read:space:confluence
 read:label:confluence
 search:confluence
@@ -98,7 +101,7 @@ read:user:confluence
 read:group:confluence
 ```
 
-Use all 12 scopes for account validation, pickers, content, permissions, and directory reads. Central indexing does not need write scopes.
+Use all 13 scopes for account validation, pickers, content, attachments, permissions, and directory reads. Central indexing does not need write scopes.
 
 4. Review and create the token, then copy it. Atlassian shows it only once.
 5. In Sim's source form, open **Service account → Add service account**. Paste the **API token**, enter **Site domain** (hostname only), and select **Add service account**. Continue in the source form with the same domain.
@@ -122,7 +125,9 @@ See Atlassian's [account setup](https://support.atlassian.com/user-management/do
 | **Filter by Label** | Optional comma-separated labels; content can match any listed label. |
 | **Metadata tags** | Labels, version, and last-modified tags. |
 
-Search manages the schedule and hides item limits. It indexes published/current content and each page's own text, including supported local callouts and code blocks. Archived content, comments, attachment contents, and expanded Include Page, Excerpt Include, or third-party macro output are excluded. Referenced pages can be indexed separately with their own permissions.
+Search manages the schedule and hides item limits. It indexes published/current content and each page's own text, including supported local callouts and code blocks. PDF, Word `.docx`, and Word 97–2003 `.doc` attachments on the selected pages and blog posts are indexed as separate documents with their parent content's permissions. Space, content-type, and label filters apply to the parent content. Attachment changes are checked on each sync, even when the parent text has not changed.
+
+Archived content, comments, other attachment formats, and expanded Include Page, Excerpt Include, or third-party macro output are excluded. Referenced pages can be indexed separately with their own permissions. Attachments over 100 MB are shown as skipped; convert older Word 6/95 files to `.docx` before attaching them.
 
 ## Manage access and sync
 
@@ -146,6 +151,7 @@ In **Sync history**, **Continuing** means a healthy listing needs another batch.
 | A new page, blog post, or label is missing | Confluence search can take time to update. Once the content appears in Confluence search with the selected label, sync again. |
 | A restricted page is missing | Both your account and the crawling account need access to the page and its ancestors. |
 | Embedded content is missing | Index the referenced page separately; remote macro output is excluded. |
+| PDF or Word attachments are missing | Check `read:attachment:confluence` and access to the parent page. Existing service-account tokens may need to be replaced with one that includes this scope. Attachment access failures are reported as a partial sync. |
 | **Reconnect** or email mismatch | Authorize with the Atlassian account matching your verified Sim email and grant all requested permissions. |
 
 Open a missing page as the affected teammate, check its space and page restrictions, then sync again after correcting access. See Atlassian's [content access](https://support.atlassian.com/confluence-cloud/docs/add-or-remove-page-restrictions/) and [permission inspection](https://support.atlassian.com/confluence-cloud/docs/inspect-a-users-permissions/) guides.

apps/docs/content/docs/search/jira.mdx

@@ -37,7 +37,9 @@ Open **Settings → Sources → Add source** and select **Jira**. This opens **A
 
 ### Choose the projects
 
-Under **Account for browsing**, select a saved account (including your personal Search account) or choose **Connect Jira account**. Enter **Jira site**, then choose **Projects**. **All** in the dropdown selects every project the account can currently browse; newly created projects are not added automatically. Clear the picker search before selecting all.
+Under **Account for browsing**, select a saved account (including your personal Search account) or choose **Connect Jira account**. Enter **Jira site**, then choose **Projects**. **All** in the dropdown includes every project the syncing account can access at each sync, including newly accessible projects. Clear the picker search before selecting all.
+
+If you selected all projects before this behavior was introduced, reselect **All** and save. Previously saved selections remain a fixed list of projects.
 
 To enter keys such as `ENG, SUPPORT` manually, use the switch beside **Projects**; this works without a browsing account. Switching between the picker and manual entry keeps your selection.
 
@@ -76,7 +78,7 @@ Each teammate follows this step. Sim reuses an existing authorized account where
 After an admin approves Jira, a teammate can select **Connect** on the Jira row in **Integrations**. To add another site or project selection later, open the Jira row's actions menu (**…**) and select **Add projects**:
 
 1. Open **Your account** and select a saved account or **Connect Jira account**. Authorize using the Atlassian email matching your verified Sim email.
-2. Enter the hostname under **Atlassian site**, then choose **Projects**. Clear the picker search to use **All**, or use the arrows beside **Projects** to enter comma-separated keys. You can select up to 1,000 projects. Changing the account or site clears the project selection.
+2. Enter the hostname under **Atlassian site**, then choose **Projects**. Clear the picker search to use **All**, or use the arrows beside **Projects** to enter comma-separated keys. You can select up to 1,000 individual projects. Changing the account or site clears the project selection.
 3. Select **Connect & Sync**. Sim saves the selected scope and starts indexing with your account.
 
 <Image className="mx-auto h-auto w-full max-w-md"

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.test.tsx

@@ -34,12 +34,68 @@ vi.mock('@/hooks/queries/selectors', () => ({
 import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field'
 
 interface ComboboxCallbacks {
-  options: { value: string; label: string; hidden?: boolean; onSelect?: () => void }[]
+  options: {
+    value: string
+    label: string
+    hidden?: boolean
+    selected?: boolean
+    onSelect?: () => void
+  }[]
   disabled: boolean
   onChange?: (value: string) => void
   onMultiSelectChange?: (value: string[]) => void
 }
 
+it('stores dynamic All without enumerating a snapshot and lets users select individual items again', async () => {
+  const field = {
+    id: 'spaces',
+    title: 'Spaces',
+    type: 'selector',
+    selectorKey: 'confluence.spaces',
+    multi: true,
+    allowSelectAll: true,
+    selectAllValue: '*',
+  } as const
+  const root = createRoot(document.createElement('div'))
+  const render = async (value: string[]) =>
+    act(async () =>
+      root.render(
+        <ConnectorSelectorField
+          field={field}
+          value={value}
+          onChange={mocks.change}
+          credentialId='credential-1'
+          sourceConfig={{ domain: 'example.atlassian.net' }}
+          configFields={[field]}
+          canonicalModes={{}}
+        />
+      )
+    )
+  try {
+    await render([])
+    await act(async () =>
+      mocks.combobox.mock
+        .lastCall![0].options.find((option) => option.label === 'All')
+        ?.onSelect?.()
+    )
+    expect(mocks.change).toHaveBeenLastCalledWith(['*'], [{ id: '*', label: 'All' }])
+    expect(mocks.loadAll).not.toHaveBeenCalled()
+    await render(['*'])
+    await act(async () => mocks.combobox.mock.lastCall![0].onMultiSelectChange?.(['*', 'folder-b']))
+    expect(mocks.change).toHaveBeenLastCalledWith(
+      ['folder-b'],
+      [{ id: 'folder-b', label: 'Company docs' }]
+    )
+    await render(['*', 'folder-b'])
+    const all = mocks.combobox.mock.lastCall![0].options.find((option) => option.label === 'All')
+    await act(async () => all?.onSelect?.())
+    expect(mocks.change).toHaveBeenLastCalledWith(['*'], [{ id: '*', label: 'All' }])
+  } finally {
+    await act(async () => root.unmount())
+    vi.clearAllMocks()
+  }
+})
+
 beforeEach(() => {
   vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true)
 })

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx

@@ -139,8 +139,11 @@ export function ConnectorSelectorField({
 
   const singleValue = Array.isArray(value) ? value[0] : value
   const selectedIds = useMemo(
-    () => (Array.isArray(value) ? value : value ? [value] : []).filter(Boolean),
-    [value]
+    () =>
+      (Array.isArray(value) ? value : value ? [value] : []).filter(
+        (id) => Boolean(id) && id !== field.selectAllValue
+      ),
+    [value, field.selectAllValue]
   )
   const missingSelectedIds = useMemo(() => {
     const loadedIds = new Set(options.map((option) => option.id))
@@ -197,6 +200,9 @@ export function ConnectorSelectorField({
   }, [options, selectedOptions, searchedOption, selectedLabels, selectedIds])
 
   const handleChange = (nextValue: ConfigFieldValue) => {
+    if (Array.isArray(nextValue) && field.selectAllValue) {
+      nextValue = nextValue.filter((id) => id !== field.selectAllValue)
+    }
     bulkGenerationRef.current += 1
     setBulkError(null)
     const ids = new Set(Array.isArray(nextValue) ? nextValue : nextValue ? [nextValue] : [])
@@ -214,18 +220,26 @@ export function ConnectorSelectorField({
 
   const hasSearch = searchTerm.trim().length > 0 || debouncedSearch.length > 0
   const selectedIdSet = new Set(selectedIds)
-  const allSelected =
-    !hasMore &&
-    !truncated &&
-    options.length > 0 &&
-    selectedIds.length === options.length &&
-    options.every((option) => selectedIdSet.has(option.id))
+  const values = Array.isArray(value) ? value : [value]
+  const allSelected = field.selectAllValue
+    ? values.length === 1 && values[0] === field.selectAllValue
+    : !hasMore &&
+      !truncated &&
+      options.length > 0 &&
+      selectedIds.length === options.length &&
+      options.every((option) => selectedIdSet.has(option.id))
   const selectAll = async () => {
     if (!isEnabled || hasSearch || isFetching || isLoadingAll) return
     if (allSelected) {
       handleChange([])
       return
     }
+    if (field.selectAllValue) {
+      bulkGenerationRef.current += 1
+      setBulkError(null)
+      onChange([field.selectAllValue], [{ id: field.selectAllValue, label: 'All' }])
+      return
+    }
     const generation = ++bulkGenerationRef.current
     setBulkError(null)
     const result = await loadAll()
@@ -265,10 +279,10 @@ export function ConnectorSelectorField({
           aria-label={field.title}
           multiSelect
           options={
-            field.allowSelectAll && (options.length > 0 || hasMore)
+            field.allowSelectAll && (options.length > 0 || hasMore || allSelected)
               ? [
                   {
-                    value: '',
+                    value: field.selectAllValue ?? '',
                     label: 'All',
                     disabled: !isEnabled || hasSearch || isFetching || isLoadingAll,
                     onSelect: () => void selectAll(),