diff --git a/apps/docs/content/docs/search/gmail.mdx b/apps/docs/content/docs/search/gmail.mdx index 97ee3fb1b5c..6eeb579613e 100644 --- a/apps/docs/content/docs/search/gmail.mdx +++ b/apps/docs/content/docs/search/gmail.mdx @@ -25,7 +25,7 @@ Open **Settings → Sources** and turn on **Gmail**. Gmail uses member accounts; ### Connect your account -Open **Integrations** and select **Connect** beside Gmail. Authorize the Google account matching your verified Sim email. The first connection creates the default sync configuration: all dates and labels, excluding Promotions, Social, Spam, and Trash. +Open **Integrations** and select **Connect** beside Gmail. Authorize the Google account matching your verified Sim email. The first connection creates the default sync configuration: the last 6 months across all labels, excluding Promotions, Social, Spam, and Trash. @@ -60,9 +60,9 @@ An admin opens **Settings → Sources**, selects **Manage** beside **Gmail**, op | Option | Behavior | | --- | --- | | Labels | Optional comma-separated names or system IDs, such as `Engineering, INBOX`. A thread matching any listed label is included. Leave empty for all labels. Custom IDs such as `Label_7` belong to one mailbox and cannot be used for member setup. | -| Date Range | All time by default. Choose the last 7, 30, or 90 days, 6 months, or year. | +| Date Range | Last 6 months by default for Search sources. Choose the last 7, 30, or 90 days, a year, or all time. A knowledge-base connector outside Search defaults to all time. | | Exclude Promotions / Exclude Social | Both enabled by default. Choose **No** to include either category. | -| Search Filter | Optional [Gmail query](https://developers.google.com/workspace/gmail/api/guides/filtering), such as `from:team@example.com subject:release`. This filters what is indexed; it is not a Sim Search query. | +| Search Filter | Optional [Gmail query](https://developers.google.com/workspace/gmail/api/guides/filtering), such as `from:team@example.com subject:release`. This filters what is indexed; it is not a Sim Search query. A source with a search filter cannot use Gmail's change history and relists the mailbox on every sync. | In the add-source form, **More options** contains optional **Metadata tags**. Sync frequency and the general knowledge-base **Max Threads** setting are hidden in Search. @@ -72,7 +72,7 @@ Sim indexes the message text Gmail returns for each matching thread, plus subjec File attachments and image contents are not indexed. Thread discovery uses Gmail's default exclusion of Spam and Trash. A filter such as `has:attachment` selects the email thread; it does not index the attachment. Gmail API filtering also differs from Gmail's interface for aliases and thread-wide searches. See Google's [thread listing reference](https://developers.google.com/workspace/gmail/api/reference/rest/v1/users.threads/list) and [filtering guide](https://developers.google.com/workspace/gmail/api/guides/filtering). -Search schedules syncs hourly. The first sync and large mailboxes can take longer; results appear as documents are indexed. Updates and removals are reconciled during background sync, rather than fetched live for each search. +Search schedules syncs hourly. The first sync lists every thread in scope and can take several runs for a large mailbox; results appear as documents are indexed. Later syncs read Gmail's change history instead of relisting the mailbox, so only threads that gained a message, were relabelled, or were deleted since the previous run are fetched. A full relisting runs about weekly, or sooner if Gmail no longer retains the history the source last read. Updates and removals are reconciled during background sync, rather than fetched live for each search. An empty mailbox or filters with no matching threads complete normally with zero documents. diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts index 49e58a34eba..43457cbc002 100644 --- a/apps/sim/connectors/gmail/gmail.test.ts +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -28,7 +28,11 @@ import { } from '@/lib/knowledge/connectors/sync-primitives' import { gmailConnector } from '@/connectors/gmail/gmail' import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta' -import { CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' +import { + CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, + memberDocumentId, + PER_MEMBER_LISTING_CONTEXT, +} from '@/connectors/utils' function threads(count: number, prefix: string) { return Array.from({ length: count }, (_, i) => ({ id: `${prefix}-${i}`, historyId: '1' })) @@ -1145,3 +1149,211 @@ describe('Gmail body and label extraction', () => { ).toHaveLength(1) }) }) + +describe('Gmail change feed', () => { + function historyPage( + threadIds: string[], + options: { nextPageToken?: string; historyId?: string } = {} + ) { + return { + historyId: options.historyId ?? '900', + nextPageToken: options.nextPageToken, + history: threadIds.map((threadId, i) => ({ + id: String(100 + i), + messages: [{ id: `m-${threadId}`, threadId }], + })), + } + } + + function metadataThread( + id: string, + messages: Array<{ labelIds: string[]; internalDate?: string }>, + historyId = '77' + ) { + return { + id, + historyId, + snippet: `Snippet ${id}`, + messages: messages.map((message, i) => ({ + id: `${id}-m${i}`, + threadId: id, + internalDate: message.internalDate ?? String(Date.now()), + labelIds: message.labelIds, + })), + } + } + + /** Routes history, label and thread reads; a thread id missing from `threads` answers 404. */ + function mockFeed( + pages: ReturnType[], + threads: Record>, + labels: Array<{ id: string; name: string }> = [{ id: 'INBOX', name: 'INBOX' }] + ) { + const requests: URL[] = [] + let historyCall = 0 + mockFetchWithRetry.mockImplementation(async (url: string) => { + const parsed = new URL(url) + requests.push(parsed) + if (parsed.pathname.endsWith('/profile')) return Response.json({ historyId: '500' }) + if (parsed.pathname.endsWith('/labels')) return Response.json({ labels }) + if (parsed.pathname.endsWith('/history')) { + return Response.json(pages[historyCall++] ?? historyPage([])) + } + const threadId = decodeURIComponent(parsed.pathname.split('/').at(-1) ?? '') + const thread = threads[threadId] + return thread ? Response.json(thread) : new Response('not found', { status: 404 }) + }) + return requests + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-09-10T12:00:00Z')) + }) + + it('opens the feed at the mailbox history id from the profile', async () => { + const requests = mockFeed([], {}) + const cursor = await gmailConnector.getChangeCursor!('token', {}) + expect(JSON.parse(cursor)).toEqual({ historyId: '500' }) + expect(requests.map((url) => url.pathname.split('/').at(-1))).toEqual(['profile']) + }) + + it('refuses the feed only when a free-form search filter is configured', () => { + expect(gmailConnector.supportsChangeFeed!({})).toBe(true) + expect(gmailConnector.supportsChangeFeed!({ label: 'INBOX', dateRange: '30d' })).toBe(true) + expect(gmailConnector.supportsChangeFeed!({ query: ' ' })).toBe(true) + expect(gmailConnector.supportsChangeFeed!({ query: 'from:boss@example.com' })).toBe(false) + }) + + it('upserts changed threads still in scope and removes trashed or deleted ones', async () => { + const requests = mockFeed([historyPage(['kept', 'trashed', 'gone'])], { + kept: metadataThread('kept', [{ labelIds: ['INBOX'] }]), + trashed: metadataThread('trashed', [{ labelIds: ['TRASH'] }, { labelIds: ['SPAM'] }]), + }) + const syncContext = memberContext('member-a') + const page = await gmailConnector.listChanges!( + 'token', + {}, + JSON.stringify({ historyId: '500' }), + syncContext + ) + + expect(page.hasMore).toBe(false) + expect(JSON.parse(page.nextCursor)).toEqual({ historyId: '900' }) + expect(page.changes).toHaveLength(3) + const kept = page.changes.find((change) => change.externalId.endsWith('kept')) + expect(kept?.kind).toBe('upsert') + if (kept?.kind !== 'upsert') throw new Error('expected an upsert') + expect(kept.document.externalId).toBe(memberDocumentId('kept', syncContext)) + expect(kept.document.contentDeferred).toBe(true) + expect(kept.document.contentHash).toBe('gmail:kept:77:body-v2') + expect( + page.changes.filter((change) => change.kind === 'removed').map((c) => c.externalId) + ).toEqual( + expect.arrayContaining([ + memberDocumentId('trashed', syncContext), + memberDocumentId('gone', syncContext), + ]) + ) + + const history = requests.find((url) => url.pathname.endsWith('/history'))! + expect(history.searchParams.get('startHistoryId')).toBe('500') + expect(history.searchParams.getAll('historyTypes')).toEqual([ + 'messageAdded', + 'messageDeleted', + 'labelAdded', + 'labelRemoved', + ]) + const threadReads = requests.filter((url) => /\/threads\/[^/]+$/.test(url.pathname)) + expect(threadReads).toHaveLength(3) + for (const read of threadReads) { + expect(read.searchParams.get('format')).toBe('metadata') + expect(read.searchParams.get('fields')).not.toContain('payload') + } + }) + + it('applies the date range, label and category filters to each message', async () => { + const dayMs = 24 * 60 * 60 * 1000 + const recent = String(Date.now() - 2 * dayMs) + const stale = String(Date.now() - 40 * dayMs) + mockFeed( + [historyPage(['old', 'promo', 'unlabelled', 'match'])], + { + old: metadataThread('old', [{ labelIds: ['Label_7'], internalDate: stale }]), + promo: metadataThread('promo', [ + { labelIds: ['Label_7', 'CATEGORY_PROMOTIONS'], internalDate: recent }, + ]), + unlabelled: metadataThread('unlabelled', [{ labelIds: ['INBOX'], internalDate: recent }]), + match: metadataThread('match', [ + { labelIds: ['INBOX'], internalDate: stale }, + { labelIds: ['Label_7'], internalDate: recent }, + ]), + }, + [{ id: 'Label_7', name: 'Engineering' }] + ) + const page = await gmailConnector.listChanges!( + 'token', + { label: 'Engineering', dateRange: '30d' }, + JSON.stringify({ historyId: '500' }) + ) + const byId = Object.fromEntries(page.changes.map((change) => [change.externalId, change.kind])) + expect(byId).toEqual({ + old: 'removed', + promo: 'removed', + unlabelled: 'removed', + match: 'upsert', + }) + }) + + it('keeps the start history id while paging and advances it once the feed drains', async () => { + const requests = mockFeed( + [ + historyPage(['a'], { nextPageToken: 'hp-2', historyId: '901' }), + historyPage(['b'], { historyId: '902' }), + ], + { + a: metadataThread('a', [{ labelIds: ['INBOX'] }]), + b: metadataThread('b', [{ labelIds: ['INBOX'] }]), + } + ) + const first = await gmailConnector.listChanges!('token', {}, '500') + expect(first.hasMore).toBe(true) + expect(JSON.parse(first.nextCursor)).toEqual({ historyId: '500', pageToken: 'hp-2' }) + + const second = await gmailConnector.listChanges!('token', {}, first.nextCursor) + expect(second.hasMore).toBe(false) + expect(JSON.parse(second.nextCursor)).toEqual({ historyId: '902' }) + + const historyReads = requests.filter((url) => url.pathname.endsWith('/history')) + expect(historyReads.map((url) => url.searchParams.get('startHistoryId'))).toEqual([ + '500', + '500', + ]) + expect(historyReads.map((url) => url.searchParams.get('pageToken'))).toEqual([null, 'hp-2']) + }) + + it('reports an expired or malformed cursor so the engine reopens from a full listing', async () => { + mockFetchWithRetry.mockImplementation( + async () => new Response('history expired', { status: 404 }) + ) + const expired = await gmailConnector.listChanges!( + 'token', + {}, + JSON.stringify({ historyId: '1' }) + ).catch((error: unknown) => error) + expect(gmailConnector.isChangeCursorInvalidError!(expired)).toBe(true) + + const malformed = await gmailConnector.listChanges!('token', {}, 'not-a-cursor').catch( + (error: unknown) => error + ) + expect(gmailConnector.isChangeCursorInvalidError!(malformed)).toBe(true) + + mockFetchWithRetry.mockImplementation(async () => new Response('boom', { status: 500 })) + const outage = await gmailConnector.listChanges!( + 'token', + {}, + JSON.stringify({ historyId: '1' }) + ).catch((error: unknown) => error) + expect(gmailConnector.isChangeCursorInvalidError!(outage)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index 1ac354eebb4..9d9fa545d97 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -5,7 +5,13 @@ import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta' -import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import type { + ConnectorConfig, + ExternalChange, + ExternalChangeList, + ExternalDocument, + ExternalDocumentList, +} from '@/connectors/types' import { BoundedLines, CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, @@ -30,6 +36,14 @@ const BODY_RESPONSE_ENVELOPE_BYTES = 1024 /** Bounds base64 bodies, alternative MIME parts, and headers before parsing the thread JSON. */ const MAX_THREAD_RESPONSE_BYTES = 32 * 1024 * 1024 +/** History records that can move a thread into or out of the configured scope. */ +const HISTORY_TYPES = ['messageAdded', 'messageDeleted', 'labelAdded', 'labelRemoved'] as const +const HISTORY_PAGE_SIZE = 500 +/** Changed threads re-read per history page before their stubs are returned. */ +const CHANGED_THREAD_CONCURRENCY = 5 +/** Gmail's thread listing omits these unless `includeSpamTrash` is set; the feed must agree. */ +const HIDDEN_LABEL_IDS = new Set(['SPAM', 'TRASH']) + class GmailApiError extends Error { constructor( message: string, @@ -261,25 +275,8 @@ function buildSearchQuery( } } - const dateRange = (sourceConfig.dateRange as string) || 'all' - const now = new Date() - switch (dateRange) { - case '7d': - parts.push(`after:${formatGmailDate(daysAgo(now, 7))}`) - break - case '30d': - parts.push(`after:${formatGmailDate(daysAgo(now, 30))}`) - break - case '90d': - parts.push(`after:${formatGmailDate(daysAgo(now, 90))}`) - break - case '6m': - parts.push(`after:${formatGmailDate(daysAgo(now, 180))}`) - break - case '1y': - parts.push(`after:${formatGmailDate(daysAgo(now, 365))}`) - break - } + const after = dateRangeStart(sourceConfig, new Date()) + if (after) parts.push(`after:${formatGmailDate(after)}`) const excludePromotions = sourceConfig.excludePromotions !== 'false' if (excludePromotions) { @@ -309,6 +306,24 @@ function buildSearchQuery( return parts.join(' ') } +const DATE_RANGE_DAYS = { + '7d': 7, + '30d': 30, + '90d': 90, + '6m': 180, + '1y': 365, +} as const + +function isBoundedDateRange(value: unknown): value is keyof typeof DATE_RANGE_DAYS { + return typeof value === 'string' && Object.hasOwn(DATE_RANGE_DAYS, value) +} + +/** The earliest message date the configured range admits, or undefined for all time. */ +function dateRangeStart(sourceConfig: Record, now: Date): Date | undefined { + const range = sourceConfig.dateRange + return isBoundedDateRange(range) ? daysAgo(now, DATE_RANGE_DAYS[range]) : undefined +} + function daysAgo(now: Date, days: number): Date { return new Date(now.getTime() - days * 24 * 60 * 60 * 1000) } @@ -537,10 +552,13 @@ async function formatThread( async function fetchThread( accessToken: string, threadId: string, - format: 'full' | 'minimal' = 'full' + format: 'full' | 'minimal' | 'metadata' = 'full' ): Promise { const params = new URLSearchParams({ format }) if (format === 'minimal') params.set('fields', 'id,historyId,snippet') + /** Enough to place every message against the configured scope without any body. */ + if (format === 'metadata') + params.set('fields', 'id,historyId,snippet,messages(id,labelIds,internalDate)') const url = `${GMAIL_API_BASE}/threads/${encodeURIComponent(threadId)}?${params}` const response = await fetchWithRetry(url, { @@ -617,11 +635,218 @@ function threadUrl(threadId: string): string { return `https://mail.google.com/mail/u/0/#all/${threadId}` } +/** A feed position: the mailbox history id the next read starts from, mid-page when paging. */ +interface GmailChangeCursor { + historyId: string + pageToken?: string +} + +class InvalidGmailChangeCursorError extends Error { + constructor() { + super('Malformed Gmail change cursor') + this.name = 'InvalidGmailChangeCursorError' + } +} + +function parseChangeCursor(cursor: string): GmailChangeCursor { + if (/^\d+$/.test(cursor)) return { historyId: cursor } + let parsed: unknown + try { + parsed = JSON.parse(cursor) + } catch { + throw new InvalidGmailChangeCursorError() + } + if ( + !isPlainRecord(parsed) || + typeof parsed.historyId !== 'string' || + !/^\d+$/.test(parsed.historyId) || + (parsed.pageToken !== undefined && + (typeof parsed.pageToken !== 'string' || parsed.pageToken.length === 0)) + ) { + throw new InvalidGmailChangeCursorError() + } + return { historyId: parsed.historyId, pageToken: parsed.pageToken as string | undefined } +} + +interface GmailHistoryList { + threadIds: string[] + nextPageToken?: string + historyId: string +} + +function parseHistoryList(value: unknown): GmailHistoryList { + if ( + !isPlainRecord(value) || + typeof value.historyId !== 'string' || + !/^\d+$/.test(value.historyId) || + (value.history !== undefined && !Array.isArray(value.history)) || + (value.nextPageToken !== undefined && + (typeof value.nextPageToken !== 'string' || value.nextPageToken.length === 0)) + ) { + throw new Error('Gmail returned malformed history metadata') + } + const threadIds = new Set() + for (const record of (value.history ?? []) as unknown[]) { + if (!isPlainRecord(record) || !Array.isArray(record.messages)) continue + for (const message of record.messages as unknown[]) { + if (isPlainRecord(message) && typeof message.threadId === 'string' && message.threadId) { + threadIds.add(message.threadId) + } + } + } + return { + threadIds: [...threadIds], + nextPageToken: value.nextPageToken as string | undefined, + historyId: value.historyId, + } +} + +/** The configured listing scope, evaluated locally against a thread's message metadata. */ +interface GmailChangeScope { + after?: Date + labelIds?: Set + excludedLabelIds: Set +} + +/** + * Mirrors {@link buildSearchQuery}: a free-form search filter cannot be + * evaluated here, which is why {@link gmailConnector.supportsChangeFeed} + * refuses the feed when one is configured. + */ +function buildChangeScope( + sourceConfig: Record, + labelIndex: GmailLabelIndex, + now: Date +): GmailChangeScope { + const scope: GmailChangeScope = { + after: dateRangeStart(sourceConfig, now), + excludedLabelIds: new Set(), + } + const configuredLabels = parseMultiValue(sourceConfig.label) + if (configuredLabels.length > 0) { + const labelIds = new Set() + for (const value of configuredLabels) { + const id = labelIndex.byId[value] ? value : labelIndex.idByLowerName[value.toLowerCase()] + if (!id) throw new Error(`Gmail label "${value}" does not exist in this mailbox`) + labelIds.add(id) + } + scope.labelIds = labelIds + } + if (sourceConfig.excludePromotions !== 'false') scope.excludedLabelIds.add('CATEGORY_PROMOTIONS') + if (sourceConfig.excludeSocial !== 'false') scope.excludedLabelIds.add('CATEGORY_SOCIAL') + return scope +} + +/** + * Gmail matches search terms per message and returns the thread of any + * matching message, so a thread is in scope while one message outside Spam and + * Trash satisfies every configured filter. + */ +function threadInScope(thread: GmailThread, scope: GmailChangeScope): boolean { + return (thread.messages ?? []).some((message) => { + const labels = new Set(message.labelIds ?? []) + for (const label of labels) { + if (HIDDEN_LABEL_IDS.has(label) || scope.excludedLabelIds.has(label)) return false + } + if (scope.labelIds && ![...scope.labelIds].some((id) => labels.has(id))) return false + if (scope.after) { + const sentAt = Number(message.internalDate) + if (!Number.isFinite(sentAt) || sentAt < scope.after.getTime()) return false + } + return true + }) +} + export const gmailConnector: ConnectorConfig = { ...gmailConnectorMeta, isCredentialInvalidError: (error) => error instanceof GmailApiError && error.status === 401, + /** The mailbox's current history id; `users.history.list` replays everything after it. */ + getChangeCursor: async (accessToken: string): Promise => { + const response = await fetchWithRetry(`${GMAIL_API_BASE}/profile?fields=historyId`, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }) + if (!response.ok) throw new GmailApiError('Failed to read the Gmail profile', response.status) + const data: unknown = await response.json() + if ( + !isPlainRecord(data) || + typeof data.historyId !== 'string' || + !/^\d+$/.test(data.historyId) + ) { + throw new Error('Gmail returned malformed profile metadata') + } + return JSON.stringify({ historyId: data.historyId }) + }, + + /** Labels, dates and categories are checked locally; a free-form search filter cannot be. */ + supportsChangeFeed: (sourceConfig) => + typeof sourceConfig.query !== 'string' || sourceConfig.query.trim() === '', + + /** + * Reads `users.history.list` from the cursor and re-reads each touched + * thread's metadata. A thread that still matches the configured scope is an + * upsert carrying the same stub a listing produces; one that was deleted, + * trashed, or relabelled out of scope is a removal. + */ + listChanges: async ( + accessToken: string, + sourceConfig: Record, + cursor: string, + syncContext?: Record + ): Promise => { + const { historyId, pageToken } = parseChangeCursor(cursor) + let labelIndex = EMPTY_LABEL_INDEX + if (parseMultiValue(sourceConfig.label).length > 0) { + const resolved = await getLabelIndex(accessToken, syncContext) + if (!resolved) { + throw new Error('Failed to fetch Gmail labels; cannot resolve the configured label filter') + } + labelIndex = resolved + } + const scope = buildChangeScope(sourceConfig, labelIndex, new Date()) + + const params = new URLSearchParams({ + startHistoryId: historyId, + maxResults: String(HISTORY_PAGE_SIZE), + fields: 'history(messages(threadId)),nextPageToken,historyId', + }) + for (const type of HISTORY_TYPES) params.append('historyTypes', type) + if (pageToken) params.set('pageToken', pageToken) + + const response = await fetchWithRetry(`${GMAIL_API_BASE}/history?${params.toString()}`, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }) + if (!response.ok) { + logger.warn('Failed to list Gmail history', { status: response.status }) + throw new GmailApiError('Failed to list Gmail history', response.status) + } + const page = parseHistoryList(await response.json()) + + const changes = await mapWithConcurrency( + page.threadIds, + CHANGED_THREAD_CONCURRENCY, + async (threadId): Promise => { + const externalId = memberDocumentId(threadId, syncContext) + const thread = await fetchThread(accessToken, threadId, 'metadata') + if (!thread || !threadInScope(thread, scope)) return { kind: 'removed', externalId } + return { kind: 'upsert', externalId, document: threadToStub(thread, syncContext) } + } + ) + + const next: GmailChangeCursor = page.nextPageToken + ? { historyId, pageToken: page.nextPageToken } + : { historyId: page.historyId } + return { changes, nextCursor: JSON.stringify(next), hasMore: Boolean(page.nextPageToken) } + }, + + /** Gmail answers 404 once `startHistoryId` falls outside the history it retains. */ + isChangeCursorInvalidError: (error) => + error instanceof InvalidGmailChangeCursorError || + (error instanceof GmailApiError && error.status === 404), + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/gmail/meta.ts b/apps/sim/connectors/gmail/meta.ts index f95d0c41534..7afa39167e6 100644 --- a/apps/sim/connectors/gmail/meta.ts +++ b/apps/sim/connectors/gmail/meta.ts @@ -19,6 +19,8 @@ export const gmailConnectorMeta: ConnectorMeta = { }, permissionScopedListing: { capFieldIds: ['maxThreads'] }, + /** A personal mailbox is indexed from the last six months unless the source says otherwise. */ + searchDefaultSourceConfig: { dateRange: '6m' }, configFields: [ { id: 'labelSelector', diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts index 836f031b831..d15113c8aa1 100644 --- a/apps/sim/connectors/types.ts +++ b/apps/sim/connectors/types.ts @@ -313,6 +313,12 @@ export interface ConnectorMeta { search?: true /** Source setup guide shown only in Search connection flows. */ searchDocsUrl?: string + /** + * Settings a Search source starts from when nobody chose a value, such as a + * bounded history window for a mailbox. Explicit settings always win, and + * knowledge-base connectors ignore these defaults. + */ + searchDefaultSourceConfig?: Readonly> /** Unique connector identifier, e.g. 'confluence', 'google_drive', 'notion' */ id: string /** Human-readable name, e.g. 'Confluence', 'Google Drive' */ diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 831b9832299..38d33c26036 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -1310,6 +1310,33 @@ describe('approved organization member source creation', () => { expect(mocks.createConnector).not.toHaveBeenCalled() }) + it('starts a personal source from the connector Search defaults and accepts their fields', async () => { + mocks.createConnector.mockResolvedValue({ + success: true, + connector: { id: 'connector', connectorType: 'gmail', accessMode: 'members' }, + }) + await createApprovedSearchSource.execute({ + principal, + input: { ...input, connectorType: 'gmail', sourceConfig: {} }, + }) + expect(mocks.createConnector).toHaveBeenLastCalledWith( + expect.objectContaining({ + sourceConfig: expect.objectContaining({ dateRange: '6m' }), + }) + ) + + queueTableRows(member, [{ role: 'member' }]) + await createApprovedSearchSource.execute({ + principal, + input: { ...input, connectorType: 'gmail', sourceConfig: { dateRange: 'all' } }, + }) + expect(mocks.createConnector).toHaveBeenLastCalledWith( + expect.objectContaining({ + sourceConfig: expect.objectContaining({ dateRange: 'all' }), + }) + ) + }) + it('refuses custom configuration outside the personal setup fields', async () => { await expect( createApprovedSearchSource.execute({ diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 70972abe4ad..c8fd8d39ca8 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -92,7 +92,11 @@ import { credentialProviderMatchesService, type ServiceProviderIdentity } from ' import { CAPABILITY_RULES, refuseCapability } from '@/lib/permission-groups/capabilities' import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' import { getUserPermissionConfigForOrganization } from '@/lib/permission-groups/resolve.server' -import { canConnectPersonally, personalSetupFields } from '@/lib/sim-search/connectors' +import { + canConnectPersonally, + personalSourceConfigFieldIds, + withSearchSourceDefaults, +} from '@/lib/sim-search/connectors' import { SIM_SEARCH_SYNC_INTERVAL_MINUTES } from '@/lib/sim-search/constants' import { describeSearchSource } from '@/lib/sim-search/source-identity' import { getConnectorApiKeyConfig, isConnectorCredentialTypeAllowed } from '@/connectors/auth' @@ -875,7 +879,7 @@ export const createApprovedSearchSource = defineAuthorizedKnowledgeUseCase({ 'Only approved personal Search sources may be connected' ) } - const allowedFields = new Set(personalSetupFields(meta).map((field) => field.id)) + const allowedFields = personalSourceConfigFieldIds(meta) if (Object.keys(input.sourceConfig).some((field) => !allowedFields.has(field))) { throw new OrchestrationError( 'validation', @@ -891,7 +895,7 @@ export const createApprovedSearchSource = defineAuthorizedKnowledgeUseCase({ knowledgeBaseId: input.knowledgeBaseId, assertedOrganizationId: input.assertedOrganizationId, connectorType: input.connectorType, - sourceConfig: input.sourceConfig, + sourceConfig: withSearchSourceDefaults(meta, input.sourceConfig), accessMode: 'members', syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, reuseSearchSource: true, diff --git a/apps/sim/lib/knowledge/application/sim-search.test.ts b/apps/sim/lib/knowledge/application/sim-search.test.ts index a0e521ba71b..78e2fbf60c7 100644 --- a/apps/sim/lib/knowledge/application/sim-search.test.ts +++ b/apps/sim/lib/knowledge/application/sim-search.test.ts @@ -97,6 +97,10 @@ vi.mock('@/lib/sim-search/connectors', () => ({ SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', canConnectPersonally: (meta: { permissionScopedListing?: unknown }) => Boolean(meta.permissionScopedListing), + withSearchSourceDefaults: ( + meta: { searchDefaultSourceConfig?: Record }, + sourceConfig: Record = {} + ) => ({ ...(meta.searchDefaultSourceConfig ?? {}), ...sourceConfig }), missingSetupFields: ( meta: { configFields: Array<{ id: string; title: string; required?: boolean }> }, sourceConfig: Record @@ -129,6 +133,14 @@ vi.mock('@/connectors/registry', () => ({ permissionScopedListing: { capFieldIds: [] }, configFields: [{ id: 'spaceKey', title: 'a space key', required: true }], }, + gmail: { + name: 'Gmail', + search: true, + auth: { mode: 'oauth', provider: 'google-email' }, + permissionScopedListing: { capFieldIds: ['maxThreads'] }, + searchDefaultSourceConfig: { dateRange: '6m' }, + configFields: [{ id: 'dateRange', title: 'Date Range', required: false }], + }, }, })) @@ -261,6 +273,56 @@ describe('connectSimSearchConnector', () => { expect(mocks.enroll).not.toHaveBeenCalled() }) + it('creates a Gmail source from the connector Search defaults when the form is untouched', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueTableRows(knowledgeBase, []) + queueTableRows(knowledgeBase, [{ id: 'kb-new' }]) + queueConnectorLookups(null, null, { + knowledgeBaseId: 'kb-new', + connectorId: 'connector-new', + } as typeof existingConnector) + + await connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'gmail' }, + }) + + expect(mocks.createConnector).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + connectorType: 'gmail', + sourceConfig: { dateRange: '6m' }, + accessMode: 'members', + }), + }) + ) + }) + + it('lets an explicit source setting replace a Search default', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueTableRows(knowledgeBase, []) + queueTableRows(knowledgeBase, [{ id: 'kb-new' }]) + queueConnectorLookups(null, null, { + knowledgeBaseId: 'kb-new', + connectorId: 'connector-new', + } as typeof existingConnector) + + await connectSimSearchConnector.execute({ + principal, + input: { + workspaceId: 'workspace-1', + connectorType: 'gmail', + sourceConfig: { dateRange: 'all' }, + }, + }) + + expect(mocks.createConnector).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ sourceConfig: { dateRange: 'all' } }), + }) + ) + }) + it('refuses before creating anything when per-member access is unavailable', async () => { mocks.resolvePermission.mockResolvedValue('admin') mocks.isMemberAccessAvailable.mockResolvedValue(false) diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts index e6940e16e02..92367573822 100644 --- a/apps/sim/lib/knowledge/application/sim-search.ts +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -50,6 +50,7 @@ import { canConnectPersonally, missingSetupFields, SIM_SEARCH_KNOWLEDGE_BASE_NAME, + withSearchSourceDefaults, } from '@/lib/sim-search/connectors' import { SIM_SEARCH_SYNC_INTERVAL_MINUTES } from '@/lib/sim-search/constants' import { searchSourceIdentity } from '@/lib/sim-search/source-identity' @@ -299,11 +300,19 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ if (context.organizationId) { await requireOrganizationSearchApproval(context.organizationId, input.connectorType) } - let target = await findSimSearchConnector({ ...input, ...owner }) + /** + * Defaults are applied before any lookup so a second person connecting with + * an untouched form lands on the source the first connection created. + */ + const sourceConfig = + input.connectorId && input.sourceConfig === undefined + ? undefined + : withSearchSourceDefaults(meta, input.sourceConfig) + let target = await findSimSearchConnector({ ...input, sourceConfig, ...owner }) if (!target) { const userId = resolvePrincipalSubjectUserId(principal) if (!userId) throw new OrchestrationError('forbidden', 'Sign in to connect your account') - const sourceConfig = input.sourceConfig ?? {} + const sourceConfig = withSearchSourceDefaults(meta, input.sourceConfig) const missing = missingSetupFields(meta, sourceConfig) if (missing.length > 0) { throw new OrchestrationError( diff --git a/apps/sim/lib/sim-search/connectors.test.ts b/apps/sim/lib/sim-search/connectors.test.ts index 254484c8cfe..51f79869aa0 100644 --- a/apps/sim/lib/sim-search/connectors.test.ts +++ b/apps/sim/lib/sim-search/connectors.test.ts @@ -122,8 +122,11 @@ import { isSearchConnectorAvailable, missingSetupFields, personalSetupFields, + personalSourceConfigFieldIds, SEARCH_CONNECTORS, + withSearchSourceDefaults, } from '@/lib/sim-search/connectors' +import { gmailConnectorMeta } from '@/connectors/gmail/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { slackConnectorMeta } from '@/connectors/slack/meta' @@ -451,3 +454,33 @@ describe('getConnectorAccessAvailability', () => { ).toEqual({ admin: false, members: false }) }) }) + +describe('withSearchSourceDefaults', () => { + it('starts a Gmail Search source from the last six months', () => { + expect(withSearchSourceDefaults(gmailConnectorMeta)).toEqual({ dateRange: '6m' }) + expect(withSearchSourceDefaults(gmailConnectorMeta, {})).toEqual({ dateRange: '6m' }) + }) + + it('keeps an explicit value and treats a blank one as untouched', () => { + expect(withSearchSourceDefaults(gmailConnectorMeta, { dateRange: 'all' })).toEqual({ + dateRange: 'all', + }) + expect( + withSearchSourceDefaults(gmailConnectorMeta, { dateRange: ' ', label: 'INBOX' }) + ).toEqual({ dateRange: '6m', label: 'INBOX' }) + }) + + it('leaves a source without Search defaults exactly as supplied', () => { + expect(withSearchSourceDefaults(googleDriveConnectorMeta, { folderId: 'f1' })).toEqual({ + folderId: 'f1', + }) + expect(withSearchSourceDefaults(googleDriveConnectorMeta)).toEqual({}) + }) + + it('lets a person supply the fields the defaults cover', () => { + expect(personalSourceConfigFieldIds(gmailConnectorMeta)).toEqual(new Set(['dateRange'])) + expect(personalSourceConfigFieldIds(googleDriveConnectorMeta)).toEqual( + new Set(personalSetupFields(googleDriveConnectorMeta).map((field) => field.id)) + ) + }) +}) diff --git a/apps/sim/lib/sim-search/connectors.ts b/apps/sim/lib/sim-search/connectors.ts index d5ce4c1c348..d1a9f22e968 100644 --- a/apps/sim/lib/sim-search/connectors.ts +++ b/apps/sim/lib/sim-search/connectors.ts @@ -120,6 +120,34 @@ export function canConnectWithDefaults(meta: ConnectorMeta): boolean { return canConnectPersonally(meta) && meta.id !== 'slack' && personalSetupFields(meta).length === 0 } +/** + * The settings a person may supply when a source is created from Sim Search: + * its setup fields plus anything the connector's Search defaults cover. + */ +export function personalSourceConfigFieldIds(meta: ConnectorMeta): Set { + return new Set([ + ...personalSetupFields(meta).map((field) => field.id), + ...Object.keys(meta.searchDefaultSourceConfig ?? {}), + ]) +} + +/** + * A Search source's settings, starting from the connector's Search defaults. + * A supplied value replaces its default; a blank one leaves the default in + * place, so an untouched form field never widens the source. + */ +export function withSearchSourceDefaults( + meta: Pick, + sourceConfig: Record = {} +): Record { + const merged: Record = { ...(meta.searchDefaultSourceConfig ?? {}) } + for (const [field, value] of Object.entries(sourceConfig)) { + if (typeof value === 'string' && value.trim() === '' && field in merged) continue + merged[field] = value + } + return merged +} + /** The setup fields a source config leaves empty. */ export function missingSetupFields( meta: ConnectorMeta,