fix(knowledge): stop connector sync failures on download-restricted Drive files and macro-only Confluence pages

apps/sim/connectors/confluence/confluence.test.ts

@@ -10,7 +10,9 @@ import {
   buildLastModifiedClause,
   confluenceConnector,
   confluenceStorageToPlainText,
+  DYNAMIC_CONTENT_SKIP_REASON,
   escapeCql,
+  extractConfluenceStorageText,
   isCurrentContent,
   preserveConfluenceCallouts,
   readIncludedLabels,
@@ -770,6 +772,77 @@ describe('confluenceStorageToPlainText', () => {
     expect(confluenceStorageToPlainText(storage)).toBe('Public body')
   })
 
+  it('keeps text authored inside legacy section and column layouts', () => {
+    const storage =
+      '<ac:structured-macro ac:name="section"><ac:rich-text-body>' +
+      '<ac:structured-macro ac:name="column"><ac:parameter ac:name="width">50%</ac:parameter>' +
+      '<ac:rich-text-body><h1>Linux Patching</h1><p>Run the playbook.</p></ac:rich-text-body>' +
+      '</ac:structured-macro>' +
+      '<ac:structured-macro ac:name="column"><ac:rich-text-body><p>Second column</p></ac:rich-text-body>' +
+      '</ac:structured-macro></ac:rich-text-body></ac:structured-macro>'
+
+    expect(confluenceStorageToPlainText(storage)).toBe(
+      'Linux Patching Run the playbook. Second column'
+    )
+  })
+
+  it('keeps page properties tables, table-macro bodies, and status labels', () => {
+    const storage =
+      '<ac:structured-macro ac:name="details"><ac:rich-text-body>' +
+      '<table><tbody><tr><th>Owner</th><td>Platform team</td></tr></tbody></table>' +
+      '</ac:rich-text-body></ac:structured-macro>' +
+      '<ac:structured-macro ac:name="table-filter"><ac:parameter ac:name="column">Name</ac:parameter>' +
+      '<ac:rich-text-body><table><tbody><tr><td>Filtered row</td></tr></tbody></table></ac:rich-text-body>' +
+      '</ac:structured-macro>' +
+      '<p>State: <ac:structured-macro ac:name="status"><ac:parameter ac:name="colour">Green</ac:parameter>' +
+      '<ac:parameter ac:name="title">Approved</ac:parameter></ac:structured-macro></p>'
+
+    expect(confluenceStorageToPlainText(storage)).toBe(
+      'Owner Platform team Filtered row State: Approved'
+    )
+  })
+
+  it('keeps new-editor panel and decision text while dropping app extensions', () => {
+    const storage =
+      '<ac:adf-extension><ac:adf-node type="panel">' +
+      '<ac:adf-attribute key="panel-type">custom</ac:adf-attribute>' +
+      '<ac:adf-content><p>Rotate the key quarterly.</p></ac:adf-content>' +
+      '</ac:adf-node><ac:adf-fallback><p>Rotate the key quarterly.</p></ac:adf-fallback></ac:adf-extension>' +
+      '<ac:adf-extension><ac:adf-node type="decision-list">' +
+      '<ac:adf-attribute key="local-id">abc</ac:adf-attribute>' +
+      '<ac:adf-node type="decision-item"><ac:adf-attribute key="state">DECIDED</ac:adf-attribute>' +
+      '<ac:adf-content>Use Vault</ac:adf-content></ac:adf-node></ac:adf-node></ac:adf-extension>' +
+      '<ac:adf-extension><ac:adf-node type="extension">' +
+      '<ac:adf-attribute key="parameters">remote</ac:adf-attribute></ac:adf-node>' +
+      '<ac:adf-fallback><p>Rendered by an app</p></ac:adf-fallback></ac:adf-extension>'
+
+    expect(confluenceStorageToPlainText(storage)).toBe(
+      '[CALLOUT] Rotate the key quarterly. Use Vault'
+    )
+  })
+
+  it('drops template placeholders and task bookkeeping but keeps task text intact', () => {
+    const storage =
+      '<p><ac:placeholder>Type your summary here</ac:placeholder></p>' +
+      '<ac:task-list><ac:task><ac:task-id>1</ac:task-id><ac:task-uuid>u</ac:task-uuid>' +
+      '<ac:task-status>incomplete</ac:task-status><ac:task-body>Ship it</ac:task-body></ac:task></ac:task-list>' +
+      '<p>Un<ac:inline-comment-marker ac:ref="r">believ</ac:inline-comment-marker>able</p>'
+
+    expect(confluenceStorageToPlainText(storage)).toBe('Ship it Unbelievable')
+  })
+
+  it('reports whether dynamic content was removed', () => {
+    expect(extractConfluenceStorageText('<p>Local</p>')).toEqual({
+      text: 'Local',
+      droppedDynamicContent: false,
+    })
+    expect(
+      extractConfluenceStorageText(
+        '<ac:structured-macro ac:name="children"><ac:parameter ac:name="depth">1</ac:parameter></ac:structured-macro>'
+      )
+    ).toEqual({ text: '', droppedDynamicContent: true })
+  })
+
   it.each(['expand', 'excerpt', 'noformat'])(
     'retains the authored content of the %s macro',
     (name) => {
@@ -902,7 +975,36 @@ describe('Confluence permission-scoped content', () => {
         cloudId: 'cloud-1',
         mirrorsSourceAcls: true,
       })
-    ).resolves.toMatchObject({ content: '', skippedExistingDisposition: 'replace' })
+    ).resolves.toMatchObject({
+      content: '',
+      skippedReason: DYNAMIC_CONTENT_SKIP_REASON,
+      skippedExistingDisposition: 'replace',
+    })
+  })
+
+  it('names dynamic-only hub pages distinctly from genuinely empty ones', async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          id: 'hub',
+          version: { number: 2 },
+          body: {
+            storage: {
+              value:
+                '<ac:structured-macro ac:name="children" /><ac:structured-macro ac:name="jira">' +
+                '<ac:parameter ac:name="jql">project = X</ac:parameter></ac:structured-macro>',
+            },
+          },
+        })
+      )
+    )
+    const document = await confluenceConnector.getDocument('token', config, 'hub', {
+      cloudId: 'cloud-1',
+      perMemberListing: true,
+      memberId: 'member-1',
+    })
+    expect(document?.skippedReason).toBe(DYNAMIC_CONTENT_SKIP_REASON)
+    expect(document?.skippedRetryPolicy).toBe('source-change')
   })
 
   it('keeps skipped pages retryable when no usable source version is available', async () => {
@@ -1015,7 +1117,7 @@ describe('Confluence permission-scoped content', () => {
       )
       const expectedHash =
         'mirrorsSourceAcls' in mode || 'perMemberListing' in mode
-          ? 'confluence:storage-local-body-v1:shared-page:1'
+          ? 'confluence:storage-local-body-v2:shared-page:1'
           : 'confluence:view-callouts:shared-page:1'
 
       expect(v2.documents[0].contentHash).toBe(expectedHash)

apps/sim/connectors/confluence/confluence.ts

@@ -93,6 +93,7 @@ const INLINE_FORMATTING_TAGS = new Set([
   'var',
   'samp',
   'time',
+  'ac:inline-comment-marker',
 ])
 
 /**
@@ -204,33 +205,98 @@ export function preserveConfluenceCallouts(html: string): string {
 }
 
 const STORAGE_MACRO_SELECTOR = 'ac\\:structured-macro, ac\\:macro'
+const ADF_NODE_SELECTOR = 'ac\\:adf-node'
+/** Callout macros whose body is prefixed with a semantic label, as on the view path. */
+const LOCAL_CALLOUT_MACROS = new Set(['info', 'note', 'warning', 'tip', 'panel'])
+/**
+ * Macros whose text is authored on the page itself: callouts, expand/excerpt/code
+ * bodies, legacy `section`/`column` layouts (which wrap the entire body of pages
+ * built in the old editor), Page Properties (`details`), table-wrapping macros,
+ * and `status` lozenges. Everything else either resolves another resource
+ * (include, jira, children, page tree, label reports) or is an app macro, and
+ * may render differently for each reader.
+ */
 const LOCAL_STORAGE_MACROS = new Set([
-  'info',
-  'note',
-  'warning',
-  'tip',
-  'panel',
+  ...LOCAL_CALLOUT_MACROS,
   'expand',
   'excerpt',
   'code',
   'noformat',
+  'section',
+  'column',
+  'details',
+  'toc-zone',
+  'chart',
+  'table-filter',
+  'table-chart',
+  'table-pivot',
+  'table-transformer',
+  'table-excerpt',
+  'table-plus',
+  'status',
 ])
+/** New-editor nodes stored as ADF whose content is authored on the page. */
+const LOCAL_ADF_NODE_TYPES = new Set(['panel', 'decision-list', 'decision-item'])
+/** ADF nodes rendered by a Forge or Connect app; their output is resolved elsewhere. */
+const APP_ADF_NODE_TYPES = new Set(['extension', 'bodiedExtension', 'inlineExtension'])
+/** Storage-format bookkeeping that is never page prose. */
+const STORAGE_NOISE_SELECTOR = [
+  'ac\\:parameter',
+  'ac\\:default-parameter',
+  'ac\\:adf-attribute',
+  'ac\\:adf-fallback',
+  'ac\\:placeholder',
+  'ac\\:task-id',
+  'ac\\:task-uuid',
+  'ac\\:task-status',
+  'script',
+  'style',
+].join(', ')
+
+/** Recorded when a scoped page holds nothing but content resolved from elsewhere. */
+export const DYNAMIC_CONTENT_SKIP_REASON =
+  'Page only contains dynamic content (child lists, includes, or app macros) that Search cannot index'
+
+export interface ConfluenceStorageText {
+  text: string
+  /** True when at least one non-local macro or app node was removed. */
+  droppedDynamicContent: boolean
+}
 
 /**
  * Search authorizes the containing page, not content expanded from another
  * resource. Read authored storage text and known local macro bodies only;
  * inclusion and third-party macros may render differently for each reader.
  */
-export function confluenceStorageToPlainText(storage: string): string {
+export function extractConfluenceStorageText(storage: string): ConfluenceStorageText {
   const $ = cheerio.load(
     storage,
     { xml: { xmlMode: false, recognizeCDATA: true, recognizeSelfClosing: true } },
     false
   )
-  $('ac\\:adf-extension').remove()
+  let droppedDynamicContent = false
+
+  for (const element of $(ADF_NODE_SELECTOR).toArray().reverse()) {
+    const node = $(element)
+    const type = node.attr('type') ?? ''
+    if (!LOCAL_ADF_NODE_TYPES.has(type)) {
+      if (APP_ADF_NODE_TYPES.has(type)) droppedDynamicContent = true
+      node.remove()
+      continue
+    }
+    const panelType = node.children('ac\\:adf-attribute[key="panel-type"]').text().trim()
+    node.children('ac\\:adf-attribute, ac\\:adf-fallback').remove()
+    const body = extractBlockJoinedText($, node)
+    const label =
+      type === 'panel'
+        ? (CALLOUT_LABELS[panelType === 'info' ? 'information' : panelType] ?? '[CALLOUT]')
+        : ''
+    node.replaceWith($('<p></p>').text([label, body].filter(Boolean).join(' ')))
+  }
 
   $(STORAGE_MACRO_SELECTOR).each((_, element) => {
     if (!LOCAL_STORAGE_MACROS.has($(element).attr('ac:name') ?? '')) {
+      droppedDynamicContent = true
       $(element).remove()
     }
   })
@@ -248,13 +314,23 @@ export function confluenceStorageToPlainText(storage: string): string {
         ? title
           ? `[CALLOUT: ${title}]`
           : '[CALLOUT]'
-        : CALLOUT_LABELS[name === 'info' ? 'information' : name]
+        : LOCAL_CALLOUT_MACROS.has(name)
+          ? CALLOUT_LABELS[name === 'info' ? 'information' : name]
+          : ''
     const text = [label, name === 'panel' ? '' : title, body].filter(Boolean).join(' ')
     macro.replaceWith($('<p></p>').text(text))
   }
 
-  $('ac\\:parameter, ac\\:default-parameter, script, style').remove()
-  return extractBlockJoinedText($, $.root()).replace(/\s+/g, ' ').trim()
+  $(STORAGE_NOISE_SELECTOR).remove()
+  return {
+    text: extractBlockJoinedText($, $.root()).replace(/\s+/g, ' ').trim(),
+    droppedDynamicContent,
+  }
+}
+
+/** Plain text of a storage-format body; see {@link extractConfluenceStorageText}. */
+export function confluenceStorageToPlainText(storage: string): string {
+  return extractConfluenceStorageText(storage).text
 }
 
 function usesPermissionScopedContent(syncContext?: Record<string, unknown>): boolean {
@@ -313,7 +389,7 @@ export function readIncludedLabels(page: Record<string, unknown>): string[] {
  * ordinary knowledge bases retain their existing rendered representation.
  */
 const CONTENT_REPRESENTATION = 'view-callouts'
-const SCOPED_CONTENT_REPRESENTATION = 'storage-local-body-v1'
+const SCOPED_CONTENT_REPRESENTATION = 'storage-local-body-v2'
 
 /**
  * Produces a canonical metadata stub with a deterministic contentHash that
@@ -690,9 +766,8 @@ export const confluenceConnector: ConnectorConfig = {
       throw new Error(`Confluence content is missing its ${bodyFormat} body`)
     }
     const rawContent = representation.value
-    const plainText = scopedContent
-      ? confluenceStorageToPlainText(rawContent)
-      : htmlToPlainText(preserveConfluenceCallouts(rawContent))
+    const scoped = scopedContent ? extractConfluenceStorageText(rawContent) : null
+    const plainText = scoped ? scoped.text : htmlToPlainText(preserveConfluenceCallouts(rawContent))
 
     const links = page._links as Record<string, unknown> | undefined
     const stub = pageToStub(
@@ -707,7 +782,12 @@ export const confluenceConnector: ConnectorConfig = {
 
     if (!plainText.trim()) {
       return {
-        ...markSkipped(stub, 'Document contains no extractable text'),
+        ...markSkipped(
+          stub,
+          scoped?.droppedDynamicContent
+            ? DYNAMIC_CONTENT_SKIP_REASON
+            : 'Document contains no extractable text'
+        ),
         skippedExistingDisposition: 'replace',
       }
     }

apps/sim/connectors/google-drive/google-drive-errors.ts

@@ -20,7 +20,18 @@ const PERMISSION_REASONS = new Set([
   'insufficientFilePermissions',
   'teamDriveMembershipRequired',
 ])
-const POLICY_REASONS = new Set(['domainPolicy', 'download_restricted_for_revision'])
+/**
+ * Owner- or admin-imposed restrictions on an otherwise readable file. The
+ * credential is valid, so these are not authorization failures; `cannotExportFile`
+ * and `cannotDownloadFile` are what Drive returns when the owner disabled
+ * download, print, and copy for viewers.
+ */
+const POLICY_REASONS = new Set([
+  'domainPolicy',
+  'download_restricted_for_revision',
+  'cannotDownloadFile',
+  'cannotExportFile',
+])
 const UNSUPPORTED_EXPORT_REASONS = new Set(['fileNotDownloadable', 'fileNotExportable'])
 const QUOTA_REASONS = new Set(['dailyLimitExceeded', 'quotaExceeded'])
 const RATE_LIMIT_REASONS = new Set([