fix(oauth): record revoked refresh tokens durably and surface them as reconnect

A refresh token the provider revoked (invalid_grant and kin) was only
flagged in Redis for an hour, so every scheduled run kept failing with a
generic error logged at ERROR three times, and the provider was asked
again each hour, indefinitely.

- account gains refresh_revoked_at/_code/_token_hash (expand-only,
  nullable). The record holds only while its hash fingerprints the stored
  refresh token, so a writer storing a new chain supersedes it; every
  reconnect path also clears it explicitly, since a provider can
  reauthorize without issuing a new refresh token.
- The record is written only on rows still holding the rejected token,
  so a refresh that lost a race to a newer chain cannot mark the live
  credential revoked. Revocations no longer use the Redis dead flag,
  which now holds only app-registration faults.
- A recorded revocation answers without the provider; one re-probe a day
  lets rejections that clear without a reconnect recover. A successful
  refresh clears it; a re-probe that fails for a passing reason keeps it.
- refreshTokenIfNeeded throws CredentialRevokedError; token resolution
  returns code OAUTH_CREDENTIAL_REVOKED (POST and GET) and logs it once at
  WARN; the executor, tool and connector sites no longer log it at ERROR.

Author: waleedlatif1

apps/sim/app/api/auth/oauth/token/route.ts

@@ -217,7 +217,10 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
     })
 
     if (!result.ok) {
-      return NextResponse.json({ error: result.error }, { status: result.status })
+      return NextResponse.json(
+        { ...(result.code ? { code: result.code } : {}), error: result.error },
+        { status: result.status }
+      )
     }
 
     return NextResponse.json(result.token, { status: 200 })

apps/sim/app/api/auth/oauth/utils.test.ts

@@ -153,7 +153,7 @@ describe('OAuth Utils', () => {
       ])
       mockRefreshOAuthToken.mockResolvedValueOnce({
         ok: false,
-        errorCode: 'token_revoked',
+        errorCode: 'invalid_client',
       })
       mockSelectChain([])
 
@@ -164,7 +164,7 @@ describe('OAuth Utils', () => {
       const installationIdentity = getOAuthRefreshCoordinationIdentity('slack:T08CM6ZNYBE')
       expect(fakeRedis.set).toHaveBeenCalledWith(
         `oauth:dead:${installationIdentity}`,
-        'token_revoked',
+        'invalid_client',
         'EX',
         3600
       )
@@ -182,7 +182,7 @@ describe('OAuth Utils', () => {
       ])
       mockRefreshOAuthToken.mockResolvedValueOnce({
         ok: false,
-        errorCode: 'token_revoked',
+        errorCode: 'invalid_client',
       })
       mockSelectChain([{ moved: new Date() }])
 

apps/sim/executor/utils/credential-token.ts

@@ -7,6 +7,7 @@ import { executeCopilotCredentialUseCase } from '@/lib/mothership/application/ex
 import { resolveCopilotOrganizationPersonalToken } from '@/lib/mothership/application/resolve-organization-personal-token'
 import { projectAssistantConnectedAccountTool } from '@/lib/mothership/assistant/connected-account-tool'
 import type { CopilotExecutionContext } from '@/lib/mothership/auth/application-delegation'
+import { CredentialRevokedError, OAUTH_CREDENTIAL_REVOKED } from '@/lib/oauth/credential-revoked'
 import {
   type CredentialTokenPayload,
   resolveCredentialAccessToken,
@@ -124,14 +125,17 @@ export async function resolveExecutorCredentialToken(
   })
 
   if (!result.ok) {
+    const message = `Failed to obtain credential for ${params.toolLabel ?? credentialId}: ${result.error}`
+    // The resolver already logged the revocation once, at WARN, where it was classified.
+    if (result.code === OAUTH_CREDENTIAL_REVOKED) {
+      throw new CredentialRevokedError(message)
+    }
     logger.error(`[${requestId}] Credential token resolution failed`, {
       status: result.status,
       credentialId,
       code: result.code,
     })
-    throw new Error(
-      `Failed to obtain credential for ${params.toolLabel ?? credentialId}: ${result.error}`
-    )
+    throw new Error(message)
   }
 
   return result.token

apps/sim/lib/auth/auth.ts

@@ -148,6 +148,7 @@ import { validateSignupEmailMx } from '@/lib/messaging/email/validation.server'
 import { isEmailVerificationEffectivelyEnabled } from '@/lib/messaging/email/verification'
 import { scheduleLifecycleEmail } from '@/lib/messaging/lifecycle'
 import { APP_ENTRY_PATH } from '@/lib/navigation/paths'
+import { clearOAuthRefreshFailure } from '@/lib/oauth/credential-service'
 import {
   getMicrosoftRefreshTokenExpiry,
   isMicrosoftProvider,
@@ -701,6 +702,26 @@ export const auth = betterAuth({
           }
         },
       },
+      update: {
+        /**
+         * Relinking an identity that already has a row updates it in place instead of creating
+         * one, and keeps the old refresh token when the provider issues none, so the fresh
+         * authorization has to clear the failures recorded against the old grant explicitly.
+         */
+        after: async (account, context) => {
+          const path = context?.path
+          if (!path?.startsWith('/oauth2/callback/') && !path?.startsWith('/callback/')) return
+          try {
+            await clearOAuthRefreshFailure(account.id)
+          } catch (error) {
+            logger.error('[account.update.after] Failed to clear recorded refresh failures', {
+              accountId: account.id,
+              providerId: account.providerId,
+              error,
+            })
+          }
+        },
+      },
     },
     session: {
       create: {

apps/sim/lib/credentials/draft-hooks.test.ts

@@ -38,6 +38,9 @@ describe('handleCreateCredentialFromDraft', () => {
     queueTableRows(schemaMock.credential, [
       { id: 'credential-existing', displayName: 'Existing Gmail' },
     ])
+    queueTableRows(schemaMock.account, [
+      { providerId: 'google-email', providerAccountId: 'subject' },
+    ])
 
     await handleCreateCredentialFromDraft({
       draft: {
@@ -99,6 +102,7 @@ describe('handleReconnectCredential', () => {
       { id: 'credential-1', accountId: null, displayName: 'Renamed Gmail' },
     ])
     queueTableRows(schemaMock.credential, [])
+    queueTableRows(schemaMock.account, [{ providerId: 'gmail', providerAccountId: 'subject-new' }])
     queueTableRows(schemaMock.account, [{ providerId: 'gmail', accountId: 'subject-new' }])
 
     await handleReconnectCredential({

apps/sim/lib/credentials/draft-hooks.ts

@@ -7,7 +7,7 @@ import { generateId } from '@sim/utils/id'
 import { and, eq, sql } from 'drizzle-orm'
 import { deleteOrphanedOAuthAccount } from '@/lib/credentials/deletion'
 import { resumeConnectorsAfterCredentialReconnect } from '@/lib/knowledge/connectors/credential-recovery'
-import { clearOAuthRefreshDeadFlag } from '@/lib/oauth/refresh-coordination'
+import { clearOAuthRefreshFailure } from '@/lib/oauth/credential-service'
 import { captureServerEvent } from '@/lib/posthog/server'
 
 const logger = createLogger('CredentialDraftHooks')
@@ -75,7 +75,7 @@ export async function handleCreateCredentialFromDraft(params: {
       .set({ updatedAt: now })
       .where(eq(schema.credential.id, existingCredential.id))
 
-    await clearOAuthRefreshDeadFlag(accountId)
+    await clearOAuthRefreshFailure(accountId)
     await resumeConnectorsAfterCredentialReconnect(accountId, now)
 
     recordAudit({
@@ -110,7 +110,7 @@ export async function handleCreateCredentialFromDraft(params: {
     accountId,
   })
 
-  await clearOAuthRefreshDeadFlag(accountId)
+  await clearOAuthRefreshFailure(accountId)
 
   recordAudit({
     workspaceId: draft.workspaceId,
@@ -210,7 +210,7 @@ export async function handleReconnectCredential(params: {
     }
   )
 
-  await clearOAuthRefreshDeadFlag(newAccountId)
+  await clearOAuthRefreshFailure(newAccountId)
   await resumeConnectorsAfterCredentialReconnect(newAccountId, now)
 
   recordAudit({

apps/sim/lib/credentials/organization-draft.test.ts

@@ -1,6 +1,7 @@
 import { pendingCredentialDraft } from '@sim/db/schema'
 import { dbChainMockFns, hasMockCondition, resetDbChainMock } from '@sim/testing'
 import { auditMock } from '@sim/testing/mocks/audit.mock'
+import { authOAuthUtilsMock } from '@sim/testing/mocks/auth-oauth-utils.mock'
 import {
   organizationMembershipMock,
   organizationMembershipMockFns,
@@ -9,7 +10,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const hoisted = vi.hoisted(() => ({
   context: vi.fn(),
-  clear: vi.fn(),
   deleteOrphan: vi.fn(),
 }))
 vi.mock('@sim/audit', () => auditMock)
@@ -18,7 +18,7 @@ vi.mock('@/lib/credentials/organization', () => ({
   getCredentialCreationOrganizationContext: hoisted.context,
 }))
 vi.mock('@/lib/credentials/deletion', () => ({ deleteOrphanedOAuthAccount: hoisted.deleteOrphan }))
-vi.mock('@/lib/oauth/refresh-coordination', () => ({ clearOAuthRefreshDeadFlag: hoisted.clear }))
+vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock)
 
 import { completeOrganizationCredentialDraft } from '@/lib/credentials/organization-draft'
 

apps/sim/lib/credentials/organization-draft.ts

@@ -9,7 +9,7 @@ import { resourceScopeCondition } from '@/lib/core/resource-scope.server'
 import { deleteOrphanedOAuthAccount } from '@/lib/credentials/deletion'
 import { getCredentialCreationOrganizationContext } from '@/lib/credentials/organization'
 import { resumeConnectorsAfterCredentialReconnect } from '@/lib/knowledge/connectors/credential-recovery'
-import { clearOAuthRefreshDeadFlag } from '@/lib/oauth/refresh-coordination'
+import { clearOAuthRefreshFailure } from '@/lib/oauth/credential-service'
 
 /** Completes the exact draft bound to the authenticated provider callback, rechecking current ownership under membership locks. */
 export async function completeOrganizationCredentialDraft(input: {
@@ -124,7 +124,7 @@ export async function completeOrganizationCredentialDraft(input: {
       oldAccountId: existing?.accountId,
     }
   })
-  await clearOAuthRefreshDeadFlag(input.accountId)
+  await clearOAuthRefreshFailure(input.accountId)
   if (result.reconnected) await resumeConnectorsAfterCredentialReconnect(input.accountId, now)
   recordAudit({
     actorId: input.userId,

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -819,18 +819,23 @@ async function resolveAccessToken(
   })
 
   if (!resolved) {
-    logger.error(`[${requestId}] Connector credential resolved no access token`, {
-      credentialId: connector.credentialId,
-      userId,
-      authMode: connectorConfig.auth.mode,
-    })
     const revocationError =
       connectorConfig.auth.mode === 'oauth' && connector.credentialId
         ? await getCredentialRevocationError(connector.credentialId)
         : null
+    const logContext = {
+      credentialId: connector.credentialId,
+      userId,
+      authMode: connectorConfig.auth.mode,
+    }
     if (revocationError && connector.credentialId) {
+      logger.warn(`[${requestId}] Connector credential was revoked by the source`, {
+        ...logContext,
+        errorCode: revocationError,
+      })
       throw new ConnectorCredentialRevokedError(connector.credentialId, revocationError)
     }
+    logger.error(`[${requestId}] Connector credential resolved no access token`, logContext)
     throw new Error(`Failed to obtain access token for credential ${connector.credentialId}`)
   }