From 785e49c56253b9c602a92b60b3995c7822492cb3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 14 Sep 2026 16:40:54 -0700 Subject: [PATCH] fix(confluence): drop scripts, styles, and unresolved Jira placeholders from indexed page text --- .../connectors/confluence/confluence.test.ts | 165 +++++++++++++++--- apps/sim/connectors/confluence/confluence.ts | 66 +++++-- .../connectors/sync-content-pass.test.ts | 6 +- 3 files changed, 194 insertions(+), 43 deletions(-) diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index f4c3601b9a9..894292d982a 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -10,15 +10,14 @@ import { buildLastModifiedClause, confluenceConnector, confluenceStorageToPlainText, + confluenceViewToPlainText, DYNAMIC_CONTENT_SKIP_REASON, escapeCql, extractConfluenceStorageText, isCurrentContent, - preserveConfluenceCallouts, readIncludedLabels, } from '@/connectors/confluence/confluence' import { extractCursor } from '@/connectors/confluence/cursor' -import { htmlToPlainText } from '@/connectors/utils' describe('Confluence service-account scopes', () => { it('requests metadata and role reads needed for complete mirrored ACLs', () => { @@ -216,14 +215,14 @@ describe('readIncludedLabels', () => { }) }) -describe('preserveConfluenceCallouts', () => { +describe('confluenceViewToPlainText', () => { it.concurrent('handles empty content', () => { - expect(preserveConfluenceCallouts('')).toBe('') + expect(confluenceViewToPlainText('')).toBe('') }) it.concurrent('leaves content with no macros unchanged', () => { const html = '

Just a normal paragraph.

' - expect(preserveConfluenceCallouts(html)).toContain('Just a normal paragraph.') + expect(confluenceViewToPlainText(html)).toContain('Just a normal paragraph.') }) it.concurrent('labels a built-in warning macro and keeps its body', () => { @@ -232,7 +231,7 @@ describe('preserveConfluenceCallouts', () => { '' + '

Do NOT use this form for GitLab access.

' + '' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('[WARNING]') expect(result).toContain('Do NOT use this form for GitLab access.') }) @@ -242,7 +241,7 @@ describe('preserveConfluenceCallouts', () => { '
' + '

Heads up.

' + '
' - expect(preserveConfluenceCallouts(html)).toContain('[INFO] Heads up.') + expect(confluenceViewToPlainText(html)).toContain('[INFO] Heads up.') }) it.concurrent('labels a built-in note macro', () => { @@ -250,7 +249,7 @@ describe('preserveConfluenceCallouts', () => { '
' + '

See also.

' + '
' - expect(preserveConfluenceCallouts(html)).toContain('[NOTE] See also.') + expect(confluenceViewToPlainText(html)).toContain('[NOTE] See also.') }) it.concurrent('labels a built-in tip macro', () => { @@ -258,7 +257,7 @@ describe('preserveConfluenceCallouts', () => { '
' + '

Pro tip.

' + '
' - expect(preserveConfluenceCallouts(html)).toContain('[TIP] Pro tip.') + expect(confluenceViewToPlainText(html)).toContain('[TIP] Pro tip.') }) it.concurrent('labels a generic custom-colored Panel macro using its header title', () => { @@ -267,7 +266,7 @@ describe('preserveConfluenceCallouts', () => { '
Do NOT use this form for:
' + '

GitLab access requests go to the private channel instead.

' + '' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('[CALLOUT: Do NOT use this form for:]') expect(result).toContain('GitLab access requests go to the private channel instead.') }) @@ -276,7 +275,7 @@ describe('preserveConfluenceCallouts', () => { const html = '
Warning:
' + '

See replacement form.

' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('[CALLOUT: Warning:] See replacement form.') }) @@ -286,7 +285,7 @@ describe('preserveConfluenceCallouts', () => { const html = '
Warning: Do not use
' + '

See replacement form.

' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('[CALLOUT: Warning: Do not use]') } ) @@ -294,13 +293,13 @@ describe('preserveConfluenceCallouts', () => { it.concurrent('falls back to a bare CALLOUT label when a Panel macro has no header text', () => { const html = '

Untitled panel body.

' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('[CALLOUT]') expect(result).toContain('Untitled panel body.') }) it.concurrent( - 'keeps the exclusion marker attached to its content through htmlToPlainText, even across surrounding whitespace collapse', + 'keeps the exclusion marker attached to its content across surrounding whitespace collapse', () => { const html = '

Intro paragraph.

\n\n' + @@ -309,7 +308,7 @@ describe('preserveConfluenceCallouts', () => { '' + '\n\n' + '

Trailing paragraph.

' - const plainText = htmlToPlainText(preserveConfluenceCallouts(html)) + const plainText = confluenceViewToPlainText(html) expect(plainText).toContain('[WARNING] Do NOT use this form for: GitLab') expect(plainText).toContain('Intro paragraph.') expect(plainText).toContain('Trailing paragraph.') @@ -325,7 +324,7 @@ describe('preserveConfluenceCallouts', () => { '

Do NOT use this form for:

' + '' + '' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).not.toContain('for:GitLab') expect(result).not.toContain('GitLabServiceNow') expect(result).toContain('Do NOT use this form for: GitLab ServiceNow') @@ -339,7 +338,7 @@ describe('preserveConfluenceCallouts', () => { '
' + '

First sentence.

Second sentence.

' + '
' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('First sentence. Second sentence.') expect(result).not.toContain('sentence.Second') } @@ -355,7 +354,7 @@ describe('preserveConfluenceCallouts', () => { '' + '
  • Outer item two
  • ' + '' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) // Each nested
  • 's text must appear exactly once, not duplicated by the // outer
  • also being matched and its .text() recursing into it. const occurrences = (result.match(/Nested item A/g) ?? []).length @@ -370,7 +369,7 @@ describe('preserveConfluenceCallouts', () => { '
    ' + '
    Cell text

    quoted text

    after quote
    ' + '
    ' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).not.toContain('quotedtext') expect(result).not.toContain('textafter') expect(result).toContain('Cell text quoted text after quote') @@ -383,7 +382,7 @@ describe('preserveConfluenceCallouts', () => { '
    ' + '

    This is unbelieveable.

    ' + '
    ' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).not.toContain('un believe able') expect(result).toContain('This is unbelieveable.') } @@ -394,7 +393,7 @@ describe('preserveConfluenceCallouts', () => { '
    ' + '

    Do not proceed!

    ' + '
    ' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).not.toContain('proceed !') expect(result).toContain('[WARNING] Do not proceed!') }) @@ -404,7 +403,7 @@ describe('preserveConfluenceCallouts', () => { '
    ' + '

    Do NOT use this form.

    ' + '
    ' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('Do NOT use this form.') }) @@ -416,7 +415,7 @@ describe('preserveConfluenceCallouts', () => { '
    Inner
    ' + '

    inner body

    ' + '' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('[CALLOUT: Outer]') expect(result).toContain('[CALLOUT: Inner] inner body') } @@ -430,7 +429,7 @@ describe('preserveConfluenceCallouts', () => { '
    ' + '

    Do not use this.

    ' + '' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).toContain('[WARNING] Do not use this.') } ) @@ -443,7 +442,7 @@ describe('preserveConfluenceCallouts', () => { '
    Inner title
    ' + '

    inner body

    ' + '' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) // The outer panel has no header of its own — it must fall back to a // bare [CALLOUT], not steal "Inner title" from the nested panel. expect(result).toContain('[CALLOUT] [CALLOUT: Inner title] inner body') @@ -455,10 +454,98 @@ describe('preserveConfluenceCallouts', () => { '
    ' + '

    Do NOT use this form for:
    GitLab

    ' + '
    ' - const result = preserveConfluenceCallouts(html) + const result = confluenceViewToPlainText(html) expect(result).not.toContain('for:GitLab') expect(result).toContain('[WARNING] Do NOT use this form for: GitLab') }) + + it.concurrent('drops app macro bootstrap scripts, inline styles, and chart data', () => { + const html = + '

    ' + + 'Colored text

    ' + + '' + + '
    ' + + '
    ' + + '' + + '

    After

    ' + expect(confluenceViewToPlainText(html)).toBe('Colored text After') + }) + + it.concurrent('keeps the word break a dropped script or style occupied', () => { + expect(confluenceViewToPlainText('

    BeforeAfter

    ')).toBe('Before After') + }) + + it.concurrent('treats a page holding only an app macro as having no text', () => { + const html = + '
    ' + + '
    ' + expect(confluenceViewToPlainText(html)).toBe('') + }) + + it.concurrent('reduces an unresolved Jira issue macro to its issue key', () => { + const html = + '

    Tracked in ' + + '' + + '' + + 'ENG-101 - ' + + 'Getting issue details... ' + + 'STATUS' + + ' and ' + + '' + + '' + + ' ENG-102 - ' + + '이슈 세부사항 가져오는 중... ' + + '상태' + + '.

    ' + expect(confluenceViewToPlainText(html)).toBe('Tracked in ENG-101 and ENG-102 .') + }) + + it.concurrent('falls back to the data attribute when the issue key link has no text', () => { + const html = + '' + + 'Getting issue details...' + + 'STATUS' + expect(confluenceViewToPlainText(html)).toBe('ENG-103') + }) + + it.concurrent('keeps the summary and status of a Jira issue macro Confluence resolved', () => { + const html = + '' + + '' + + 'OPS-201 - ' + + 'Rotate the signing key ' + + 'Done' + + '' + expect(confluenceViewToPlainText(html)).toBe('OPS-201 - Rotate the signing key Done') + }) + + it.concurrent('keeps the block break of a Jira issue macro inside a callout', () => { + const html = + '
    ' + + '
    Blocked by' + + '
    ' + + 'ENG-104 - ' + + 'Getting issue details...' + + 'STATUS
    ' + + 'until release
    ' + expect(confluenceViewToPlainText(html)).toBe('[WARNING] Blocked by ENG-104 until release') + }) + + it.concurrent('drops only the placeholder shell of a Jira issues table', () => { + const html = + '

    Release notes

    ' + + '
    ' + + '
    ' + + '
    typekeysummary
    ' + + '
    Loading...
    ' + + '
    ' + + '' + + '
    keysummary
    ENG-104Update the runbook
    ' + + '
    No issues found
    ' + expect(confluenceViewToPlainText(html)).toBe( + 'Release notes key summary ENG-104 Update the runbook No issues found' + ) + }) }) describe('confluence incremental CQL listing', () => { @@ -1055,7 +1142,29 @@ describe('Confluence permission-scoped content', () => { ) expect(document?.content).toContain('CONFIDENTIAL SALARY DATA') - expect(document?.contentHash).toBe('confluence:view-callouts:shared-page:1') + expect(document?.contentHash).toBe('confluence:view-text-v2:shared-page:1') + }) + + it('reports the content type of the endpoint that answered and omits unknown metadata', async () => { + vi.mocked(fetch).mockResolvedValueOnce(new Response('', { status: 404 })) + + const document = await confluenceConnector.getDocument('token', config, 'shared-page', { + cloudId: 'cloud-1', + }) + + expect(vi.mocked(fetch).mock.calls.map(([input]) => new URL(String(input)).pathname)).toEqual([ + '/ex/confluence/cloud-1/wiki/api/v2/pages/shared-page', + '/ex/confluence/cloud-1/wiki/api/v2/blogposts/shared-page', + ]) + expect(document?.metadata).toEqual({ + spaceId: 'space-1', + contentType: 'blogpost', + status: 'current', + version: 1, + labels: [], + lastModified: '', + }) + expect(document?.metadata).not.toHaveProperty('spaceKey') }) it('rejects a missing storage body without falling back to rendered content', async () => { @@ -1118,7 +1227,7 @@ describe('Confluence permission-scoped content', () => { const expectedHash = 'mirrorsSourceAcls' in mode || 'perMemberListing' in mode ? 'confluence:storage-local-body-v2:shared-page:1' - : 'confluence:view-callouts:shared-page:1' + : 'confluence:view-text-v2:shared-page:1' expect(v2.documents[0].contentHash).toBe(expectedHash) expect(cql.documents[0].contentHash).toBe(expectedHash) diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 263d50c7db0..f0da25bdb44 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { filterUndefined } from '@sim/utils/object' import * as cheerio from 'cheerio' import { AtlassianSiteNotAccessibleError, @@ -172,9 +173,37 @@ function extractBlockJoinedText($: cheerio.CheerioAPI, $el: cheerio.Cheerio return parts.join(' ').trim() } -/** Matches either flavor of panel/macro this function rewrites. */ +/** Matches either flavor of panel/macro {@link rewriteConfluenceCallouts} rewrites. */ const MACRO_SELECTOR = 'div.confluence-information-macro, div.panel' +/** + * Rendered-page elements whose text is never page prose. App macros render as a + * bootstrap `