diff --git a/apps/sim/app/api/speech/token/route.test.ts b/apps/sim/app/api/speech/token/route.test.ts index 2042c24ec22..1e0a602b067 100644 --- a/apps/sim/app/api/speech/token/route.test.ts +++ b/apps/sim/app/api/speech/token/route.test.ts @@ -1,129 +1,296 @@ -/** - * @vitest-environment node - */ +/** @vitest-environment node */ +import * as workspaceAuthz from '@sim/platform-authz/workspace' import { authMockFns, createMockRequest, + dbChainMockFns, + envFlagsMockFns, resetDbChainMock, + resetEnvFlagsMock, resetEnvMock, setEnv, + setEnvFlags, } from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockRecordUsage, - mockVerifyWorkspaceMembership, - mockResolveBillingAttribution, - mockCheckAttributedUsageLimits, - mockToBillingContext, - mockCheckAndBillPayerOverageThreshold, -} = vi.hoisted(() => ({ - mockRecordUsage: vi.fn(), - mockVerifyWorkspaceMembership: vi.fn(), - mockResolveBillingAttribution: vi.fn(), - mockCheckAttributedUsageLimits: vi.fn(), - mockToBillingContext: vi.fn(), - mockCheckAndBillPayerOverageThreshold: vi.fn(), -})) +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/billing/core/usage-log', () => ({ recordUsage: mockRecordUsage })) +const mocks = vi.hoisted(() => ({ + recordUsage: vi.fn(), + resolveBilling: vi.fn(), + resolveOrganizationBilling: vi.fn(), + checkUsage: vi.fn(), + toBillingContext: vi.fn(), + billOverage: vi.fn(), + rateCheck: vi.fn(), + organizationConfig: vi.fn(), + workspaceContext: vi.fn(), +})) +vi.mock('@/lib/billing/core/usage-log', () => ({ recordUsage: mocks.recordUsage })) vi.mock('@/lib/billing/core/billing-attribution', () => ({ - resolveBillingAttribution: mockResolveBillingAttribution, - checkAttributedUsageLimits: mockCheckAttributedUsageLimits, - toBillingContext: mockToBillingContext, + resolveBillingAttribution: mocks.resolveBilling, + resolveOrganizationBillingAttribution: mocks.resolveOrganizationBilling, + checkAttributedUsageLimits: mocks.checkUsage, + toBillingContext: mocks.toBillingContext, })) - vi.mock('@/lib/billing/threshold-billing', () => ({ - checkAndBillPayerOverageThreshold: mockCheckAndBillPayerOverageThreshold, -})) - -vi.mock('@/app/api/workflows/utils', () => ({ - verifyWorkspaceMembership: mockVerifyWorkspaceMembership, + checkAndBillPayerOverageThreshold: mocks.billOverage, })) - vi.mock('@/lib/core/rate-limiter', () => ({ RateLimiter: class { - checkRateLimitDirect = vi.fn().mockResolvedValue({ allowed: true }) + checkRateLimitDirect = mocks.rateCheck }, })) +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfigForOrganization: mocks.organizationConfig, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + resolveActiveWorkspaceApplicationContext: mocks.workspaceContext, +})) +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { createSpeechToken } from '@/lib/speech/application/create-token' import { POST } from '@/app/api/speech/token/route' -const mockGetSession = authMockFns.mockGetSession +const permission = vi.spyOn(workspaceAuthz, 'resolveEffectiveWorkspacePermission') +const principal = { kind: 'session', userId: 'member-1', sessionId: 'session-1' } as const +const billingEntity = { type: 'organization', id: 'org-1' } as const +const billingPeriod = { start: new Date('2026-07-01'), end: new Date('2026-08-01') } beforeEach(() => { vi.clearAllMocks() resetDbChainMock() setEnv({ ELEVENLABS_API_KEY: 'test-key' }) - mockGetSession.mockResolvedValue({ user: { id: 'member-1' } }) - mockRecordUsage.mockResolvedValue(undefined) - mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false }) - mockResolveBillingAttribution.mockImplementation( - ({ actorUserId, workspaceId }: { actorUserId: string; workspaceId: string }) => ({ - actorUserId, - workspaceId, - billingEntity: { type: 'organization', id: 'org-1' }, - }) - ) - mockToBillingContext.mockImplementation( - (attribution: { billingEntity: { type: 'organization' | 'user'; id: string } }) => ({ - billingEntity: attribution.billingEntity, - billingPeriod: { - start: new Date('2026-07-01T00:00:00.000Z'), - end: new Date('2026-08-01T00:00:00.000Z'), - }, - }) - ) - mockVerifyWorkspaceMembership.mockResolvedValue('admin') - global.fetch = vi.fn().mockResolvedValue({ - ok: true, - json: async () => ({ token: 'tok-123' }), - // double-cast-allowed: minimal fetch stub for the ElevenLabs token call - }) as unknown as typeof fetch + setEnvFlags({ isBillingEnabled: true }) + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: principal.userId }, + session: { id: principal.sessionId }, + }) + permission.mockResolvedValue('read') + mocks.workspaceContext.mockImplementation(async (workspaceId: string) => ({ + workspaceId, + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'owner-1', + })) + mocks.organizationConfig.mockResolvedValue(null) + dbChainMockFns.limit.mockResolvedValue([{ role: 'member' }]) + mocks.recordUsage.mockResolvedValue(undefined) + mocks.billOverage.mockResolvedValue(undefined) + mocks.rateCheck.mockResolvedValue({ allowed: true }) + mocks.checkUsage.mockResolvedValue({ isExceeded: false }) + mocks.resolveBilling.mockImplementation(async (input) => ({ ...input, billingEntity })) + mocks.resolveOrganizationBilling.mockImplementation(async (input) => ({ + ...input, + workspaceId: null, + billedAccountUserId: 'owner-1', + billingEntity, + })) + mocks.toBillingContext.mockReturnValue({ billingEntity, billingPeriod }) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json({ token: 'tok-123' }))) }) -afterAll(() => { +afterEach(() => { + vi.unstubAllGlobals() resetDbChainMock() resetEnvMock() + resetEnvFlagsMock() }) -describe('POST /api/speech/token — usage attribution', () => { - it('editor voice: bills the session user and stamps the verified workspace', async () => { - const res = await POST(createMockRequest('POST', { workspaceId: 'ws-1' })) +describe('POST /api/speech/token', () => { + it.each(['read', 'write', 'admin'] as const)( + 'allows workspace %s members and bills the acting user', + async (role) => { + permission.mockResolvedValue(role) + envFlagsMockFns.getCostMultiplier.mockReturnValue(2) + const response = await POST(createMockRequest('POST', { workspaceId: 'ws-1' })) - expect(res.status).toBe(200) - expect(mockVerifyWorkspaceMembership).toHaveBeenCalledWith('member-1', 'ws-1') - expect(mockRecordUsage).toHaveBeenCalledTimes(1) - expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({ - userId: 'member-1', - workspaceId: 'ws-1', - }) - expect(mockResolveBillingAttribution).toHaveBeenCalledWith({ - actorUserId: 'member-1', - workspaceId: 'ws-1', + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ token: 'tok-123' }) + expect(permission).toHaveBeenCalledWith('member-1', 'ws-1', null, undefined, { + forUpdate: undefined, + }) + expect(mocks.resolveBilling).toHaveBeenCalledWith({ + actorUserId: 'member-1', + workspaceId: 'ws-1', + }) + expect(mocks.recordUsage).toHaveBeenCalledWith({ + userId: 'member-1', + workspaceId: 'ws-1', + billingEntity, + billingPeriod, + entries: [ + { + category: 'fixed', + source: 'voice-input', + description: 'Voice input session (3 min)', + cost: 0.048, + sourceReference: expect.stringMatching(/^voice-input:[a-f0-9]{64}$/), + }, + ], + }) + expect(mocks.billOverage).toHaveBeenCalledWith(billingEntity) + } + ) + + it.each(['member', 'admin', 'owner'])( + 'allows organization %s members without inventing a workspace', + async (role) => { + dbChainMockFns.limit.mockResolvedValue([{ role }]) + const response = await POST(createMockRequest('POST', { organizationId: 'org-1' })) + + expect(response.status).toBe(200) + expect(mocks.resolveOrganizationBilling).toHaveBeenCalledWith({ + actorUserId: 'member-1', + organizationId: 'org-1', + }) + expect(mocks.resolveBilling).not.toHaveBeenCalled() + expect(mocks.workspaceContext).not.toHaveBeenCalled() + expect(mocks.recordUsage.mock.calls[0][0]).toMatchObject({ + userId: 'member-1', + billingEntity, + billingPeriod, + }) + expect(mocks.recordUsage.mock.calls[0][0]).not.toHaveProperty('workspaceId') + } + ) + + it('authenticates before reading an oversized body', async () => { + authMockFns.mockGetSession.mockResolvedValue(null) + const response = await POST(createMockRequest('POST', { workspaceId: 'x'.repeat(64 * 1024) })) + expect(response.status).toBe(401) + expect(mocks.rateCheck).not.toHaveBeenCalled() + expect(mocks.workspaceContext).not.toHaveBeenCalled() + }) + + it('caps authenticated bodies before protected loading', async () => { + const response = await POST(createMockRequest('POST', { workspaceId: 'x'.repeat(64 * 1024) })) + expect(response.status).toBe(413) + expect(mocks.workspaceContext).not.toHaveBeenCalled() + expect(mocks.recordUsage).not.toHaveBeenCalled() + }) + + it.each([ + {}, + { workspaceId: 'ws-1', organizationId: 'org-1' }, + { organizationId: '' }, + { workspaceId: 1 }, + ])('rejects absent, ambiguous or invalid scope %j', async (body) => { + const response = await POST(createMockRequest('POST', body)) + expect(response.status).toBe(400) + expect(mocks.workspaceContext).not.toHaveBeenCalled() + expect(mocks.resolveOrganizationBilling).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('conceals a workspace the caller cannot access', async () => { + permission.mockResolvedValue(null) + const response = await POST(createMockRequest('POST', { workspaceId: 'ws-other' })) + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: 'Workspace or organization context is required.', }) - expect(mockCheckAndBillPayerOverageThreshold).toHaveBeenCalledWith({ - type: 'organization', - id: 'org-1', + expect(mocks.resolveBilling).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('rejects removed organization members before billing or token creation', async () => { + dbChainMockFns.limit.mockResolvedValue([]) + const response = await POST(createMockRequest('POST', { organizationId: 'org-other' })) + expect(response.status).toBe(400) + expect(mocks.resolveOrganizationBilling).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('rejects inactive workspaces before billing', async () => { + mocks.workspaceContext.mockRejectedValue( + new OrchestrationError('not_found', 'Workspace not found') + ) + const response = await POST(createMockRequest('POST', { workspaceId: 'ws-archived' })) + expect(response.status).toBe(400) + expect(mocks.resolveBilling).not.toHaveBeenCalled() + }) + + it('rates by actor with the existing bucket and retry header before parsing', async () => { + mocks.rateCheck.mockResolvedValue({ allowed: false, retryAfterMs: 1501 }) + const response = await POST(createMockRequest('POST', {})) + expect(response.status).toBe(429) + expect(response.headers.get('Retry-After')).toBe('2') + expect(mocks.rateCheck).toHaveBeenCalledWith('stt-token:user:member-1', { + maxTokens: 30, + refillRate: 3, + refillIntervalMs: 72000, }) + expect(mocks.resolveOrganizationBilling).not.toHaveBeenCalled() }) - it('editor voice: rejects an unverified workspace id (requires an attributable workspace)', async () => { - mockVerifyWorkspaceMembership.mockResolvedValue(null) + it('preserves the rate exemption when billing is disabled', async () => { + setEnvFlags({ isBillingEnabled: false }) + const response = await POST(createMockRequest('POST', { organizationId: 'org-1' })) + expect(response.status).toBe(200) + expect(mocks.rateCheck).not.toHaveBeenCalled() + }) - const res = await POST(createMockRequest('POST', { workspaceId: 'ws-not-mine' })) + it.each(['actor', 'payer', 'member'])( + 'enforces the %s usage cap before contacting the provider', + async (scope) => { + mocks.checkUsage.mockResolvedValue({ isExceeded: true, message: 'Usage cap reached', scope }) + const response = await POST(createMockRequest('POST', { organizationId: 'org-1' })) + expect(response.status).toBe(402) + expect(await response.json()).toMatchObject({ error: 'Usage cap reached', scope }) + expect(fetch).not.toHaveBeenCalled() + expect(mocks.recordUsage).not.toHaveBeenCalled() + } + ) - expect(res.status).toBe(400) - expect(mockRecordUsage).not.toHaveBeenCalled() + it('does not conceal membership infrastructure failures as missing membership', async () => { + dbChainMockFns.limit.mockRejectedValue(new Error('database unavailable')) + const response = await POST(createMockRequest('POST', { organizationId: 'org-1' })) + expect(response.status).toBe(500) + expect(mocks.resolveOrganizationBilling).not.toHaveBeenCalled() }) - it('rejects an oversized body before any auth/billing work runs', async () => { - const oversizedBody = { workspaceId: 'x'.repeat(64 * 1024) } - const res = await POST(createMockRequest('POST', oversizedBody)) + it('does not issue tokens after billing attribution fails', async () => { + mocks.resolveOrganizationBilling.mockRejectedValue(new Error('billing unavailable')) + const response = await POST(createMockRequest('POST', { organizationId: 'org-1' })) + expect(response.status).toBe(500) + expect(fetch).not.toHaveBeenCalled() + }) + + it('preserves service-not-configured and upstream errors', async () => { + setEnv({ ELEVENLABS_API_KEY: '' }) + expect((await POST(createMockRequest('POST', { organizationId: 'org-1' }))).status).toBe(503) + setEnv({ ELEVENLABS_API_KEY: 'key' }) + vi.mocked(fetch).mockResolvedValue( + Response.json({ detail: 'Provider unavailable' }, { status: 503 }) + ) + const response = await POST(createMockRequest('POST', { organizationId: 'org-1' })) + expect(response.status).toBe(502) + expect(await response.json()).toMatchObject({ error: 'Provider unavailable' }) + expect(mocks.recordUsage).not.toHaveBeenCalled() + }) + + it.each(['recordUsage', 'billOverage'] as const)( + 'keeps an issued token available when %s fails', + async (failure) => { + mocks[failure].mockRejectedValue(new Error('billing write failed')) + const response = await POST(createMockRequest('POST', { organizationId: 'org-1' })) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ token: 'tok-123' }) + if (failure === 'recordUsage') expect(mocks.billOverage).not.toHaveBeenCalled() + } + ) - expect(res.status).toBe(413) - expect(mockGetSession).not.toHaveBeenCalled() - expect(mockRecordUsage).not.toHaveBeenCalled() + it('rejects non-session principals before any protected loading', async () => { + for (const input of [{ workspaceId: 'ws-1' }, { organizationId: 'org-1' }]) { + await expect( + createSpeechToken.execute({ + principal: { kind: 'personal_api_key', userId: 'member-1', keyId: 'key-1' }, + input, + }) + ).rejects.toThrow('cannot perform operation speech.token.create') + } + expect(mocks.workspaceContext).not.toHaveBeenCalled() + expect(dbChainMockFns.limit).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/app/api/speech/token/route.ts b/apps/sim/app/api/speech/token/route.ts index 6955d97b2c8..b290967287d 100644 --- a/apps/sim/app/api/speech/token/route.ts +++ b/apps/sim/app/api/speech/token/route.ts @@ -1,172 +1,80 @@ -import { createHash } from 'node:crypto' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { speechTokenBodySchema } from '@/lib/api/contracts/media/speech' -import { parseOptionalJsonBody } from '@/lib/api/server' -import { getSession } from '@/lib/auth' +import { requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { NextResponse } from 'next/server' +import { speechTokenContract } from '@/lib/api/contracts/media/speech' import { - checkAttributedUsageLimits, - resolveBillingAttribution, - toBillingContext, -} from '@/lib/billing/core/billing-attribution' -import { recordUsage } from '@/lib/billing/core/usage-log' -import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' -import { env } from '@/lib/core/config/env' -import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' + defineInternalJsonRoute, + internalErrorResponse, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { NoWorkspaceAccessError } from '@/lib/core/application/workspace-authorization' +import { isBillingEnabled } from '@/lib/core/config/env-flags' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { RateLimiter } from '@/lib/core/rate-limiter' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' - -const logger = createLogger('SpeechTokenAPI') +import { + createSpeechToken, + SpeechTokenError, + speechTokenOperation, +} from '@/lib/speech/application/create-token' export const dynamic = 'force-dynamic' -const ELEVENLABS_TOKEN_URL = 'https://api.elevenlabs.io/v1/single-use-token/realtime_scribe' - -const VOICE_SESSION_COST_PER_MIN = 0.008 -const WORKSPACE_SESSION_MAX_MINUTES = 3 - const STT_TOKEN_RATE_LIMIT = { maxTokens: 30, refillRate: 3, refillIntervalMs: 72 * 1000, } as const - -/** - * This body only ever carries an optional workspaceId string, so a - * tight cap keeps an unauthenticated caller from forcing a large in-memory - * allocation before the auth checks below run. - */ -const MAX_SPEECH_TOKEN_BODY_BYTES = 16 * 1024 - -function hashVoiceToken(token: string): string { - return createHash('sha256').update(token).digest('hex') -} - const rateLimiter = new RateLimiter() -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const parsedBody = await parseOptionalJsonBody(request, MAX_SPEECH_TOKEN_BODY_BYTES) - if (!parsedBody.success) return parsedBody.response - const body = speechTokenBodySchema.safeParse(parsedBody.data ?? {}) - - let workspaceId: string | undefined - - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const actorUserId = session.user.id - /** - * Accepts only a workspace the caller belongs to, preventing client-supplied - * IDs from misattributing or bypassing member usage. - */ - const requestedWorkspaceId = - body.success && typeof body.data.workspaceId === 'string' ? body.data.workspaceId : undefined - if (requestedWorkspaceId) { - const permission = await verifyWorkspaceMembership(session.user.id, requestedWorkspaceId) - if (permission) workspaceId = requestedWorkspaceId - } - /** - * Workspace-scoped so every charge has a payer and member cap attribution. - */ - if (!workspaceId) { - return NextResponse.json({ error: 'Workspace context is required.' }, { status: 400 }) - } - - const billingAttribution = await resolveBillingAttribution({ actorUserId, workspaceId }) - - if (isBillingEnabled) { +export const POST = defineInternalJsonRoute({ + contract: speechTokenContract, + auth: internalSessionAuth, + operation: speechTokenOperation, + rateLimit: { + kind: 'user', + bucketName: 'stt-token', + async enforce(_request, principal) { + if (!isBillingEnabled) return null const rateCheck = await rateLimiter.checkRateLimitDirect( - `stt-token:user:${actorUserId}`, + `stt-token:user:${requirePrincipalSubjectUserId(principal)}`, STT_TOKEN_RATE_LIMIT ) - if (!rateCheck.allowed) { - return NextResponse.json( - { error: 'Voice input rate limit exceeded. Please try again later.' }, - { - status: 429, - headers: { - 'Retry-After': String(Math.ceil((rateCheck.retryAfterMs ?? 60000) / 1000)), - }, - } - ) + return rateCheck.allowed + ? null + : NextResponse.json( + { error: 'Voice input rate limit exceeded. Please try again later.' }, + { + status: 429, + headers: { + 'Retry-After': String(Math.ceil((rateCheck.retryAfterMs ?? 60000) / 1000)), + }, + } + ) + }, + }, + parseOptions: { + maxBodyBytes: 16 * 1024, + validationErrorResponse: () => + NextResponse.json( + { error: 'Workspace or organization context is required.' }, + { status: 400 } + ), + }, + errorPolicy: { + project(error) { + if (error instanceof SpeechTokenError) { + const status = { usage_limit: 402, unconfigured: 503, provider_failed: 502 }[error.reason] + return internalErrorResponse(status, { error: error.message, scope: error.scope }) } - } - - /** - * Read-then-act, as every metered route in the repo does: concurrent calls - * can pass against the same balance and overshoot the payer's limit. Making - * this atomic needs a reservation primitive in `lib/billing` that fixes the - * class everywhere, not a one-off here. - * - * Accepted as bounded rather than eliminated: this route is session-gated - * and rate limited per user, so the overshoot is a knowable ceiling against - * an identified payer. - */ - const usageCheck = await checkAttributedUsageLimits(billingAttribution) - if (usageCheck.isExceeded) { - return NextResponse.json( - { - error: - usageCheck.message || 'Usage limit exceeded. Please upgrade your plan to continue.', - scope: usageCheck.scope, - }, - { status: 402 } - ) - } - - const apiKey = env.ELEVENLABS_API_KEY - if (!apiKey?.trim()) { - return NextResponse.json( - { error: 'Speech-to-text service is not configured' }, - { status: 503 } - ) - } - - const response = await fetch(ELEVENLABS_TOKEN_URL, { - method: 'POST', - headers: { 'xi-api-key': apiKey }, - }) - - if (!response.ok) { - const errBody = await response.json().catch(() => ({})) - const message = - errBody.detail || errBody.message || `Token request failed (${response.status})` - logger.error('ElevenLabs token request failed', { status: response.status, message }) - return NextResponse.json({ error: message }, { status: 502 }) - } - - const data = await response.json() - - const sessionCost = VOICE_SESSION_COST_PER_MIN * WORKSPACE_SESSION_MAX_MINUTES - - try { - await recordUsage({ - userId: actorUserId, - workspaceId, - ...toBillingContext(billingAttribution), - entries: [ - { - category: 'fixed', - source: 'voice-input', - description: `Voice input session (${WORKSPACE_SESSION_MAX_MINUTES} min)`, - cost: sessionCost * getCostMultiplier(), - sourceReference: `voice-input:${hashVoiceToken(data.token)}`, - }, - ], - }) - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } catch (err) { - logger.warn('Failed to record voice input usage, continuing:', err) - } - - return NextResponse.json({ token: data.token }) - } catch (error) { - const message = getErrorMessage(error, 'Failed to generate speech token') - logger.error('Speech token error:', error) - return NextResponse.json({ error: message }, { status: 500 }) - } + const classified = asOrchestrationError(error) + if (error instanceof NoWorkspaceAccessError || classified?.code === 'not_found') { + return internalErrorResponse(400, { + error: 'Workspace or organization context is required.', + }) + } + return null + }, + unhandled: () => internalErrorResponse(500, { error: 'Failed to generate speech token' }), + }, + mapInput: ({ body }) => body, + useCase: createSpeechToken, }) diff --git a/apps/sim/app/o/[organizationId]/home/components/composer/composer.test.tsx b/apps/sim/app/o/[organizationId]/home/components/composer/composer.test.tsx new file mode 100644 index 00000000000..b8e873529b2 --- /dev/null +++ b/apps/sim/app/o/[organizationId]/home/components/composer/composer.test.tsx @@ -0,0 +1,111 @@ +/** @vitest-environment jsdom */ +import { act, useState } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { useSpeechToText } from '@/hooks/use-speech-to-text' + +const mocks = vi.hoisted(() => ({ + speech: vi.fn(), + toggleListening: vi.fn(), + resetTranscript: vi.fn(), + submit: vi.fn(), +})) + +vi.mock('@/hooks/use-speech-to-text', () => ({ useSpeechToText: mocks.speech })) +vi.mock('@/hooks/use-animated-placeholder', () => ({ useAnimatedPlaceholder: () => 'Ask Sim to' })) +vi.mock('@/hooks/use-chat-input-focus', () => ({ useChatInputFocus: vi.fn() })) +vi.mock('@/app/o/[organizationId]/providers/organization-provider', () => ({ + useOrganizationContext: () => ({ organization: { id: 'organization-a' } }), +})) + +import { Composer } from '@/app/o/[organizationId]/home/components/composer/composer' + +let root: Root +let container: HTMLDivElement + +beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.stubGlobal( + 'matchMedia', + vi.fn(() => ({ + matches: false, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + })) + ) + mocks.speech.mockReturnValue({ + isSupported: true, + isListening: false, + audioLevelsRef: { current: new Float32Array(5) }, + toggleListening: mocks.toggleListening, + resetTranscript: mocks.resetTranscript, + }) + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) +}) + +afterEach(async () => { + await act(async () => root.unmount()) + container.remove() + vi.unstubAllGlobals() +}) + +async function render(isInitialView: boolean) { + function Harness() { + const [value, setValue] = useState('Summarize') + return ( + { + mocks.submit(value) + setValue('') + }} + /> + ) + } + await act(async () => root.render()) +} + +describe('organization voice composer', () => { + it.each([true, false])( + 'appends dictation and clears its prefix on send (initial: %s)', + async (isInitialView) => { + await render(isInitialView) + const mic = container.querySelector('button[aria-label="Voice input"]')! + expect(mic.nextElementSibling?.getAttribute('aria-label')).toBe('Send') + await act(async () => mic.click()) + expect(mocks.toggleListening).toHaveBeenCalledOnce() + const speech = mocks.speech.mock.calls.at(-1)![0] + expect(speech.organizationId).toBe('organization-a') + await act(async () => speech.onTranscript('the')) + await act(async () => speech.onTranscript('the release')) + expect(container.querySelector('textarea')!.value).toBe('Summarize the release') + expect(mocks.submit).not.toHaveBeenCalled() + await act(async () => { + container.querySelector('button[aria-label="Send"]')!.click() + }) + expect(mocks.submit).toHaveBeenCalledWith('Summarize the release') + expect(mocks.resetTranscript).toHaveBeenCalledOnce() + await act(async () => mocks.speech.mock.calls.at(-1)![0].onTranscript('Next question')) + expect(container.querySelector('textarea')!.value).toBe('Next question') + } + ) + + it('hides voice input when unavailable', async () => { + mocks.speech.mockImplementation(() => ({ + isSupported: false, + isListening: false, + audioLevelsRef: { current: new Float32Array(5) }, + toggleListening: mocks.toggleListening, + resetTranscript: mocks.resetTranscript, + })) + await render(true) + expect(container.querySelector('button[aria-label="Voice input"]')).toBeNull() + }) +}) diff --git a/apps/sim/app/o/[organizationId]/home/components/composer/composer.tsx b/apps/sim/app/o/[organizationId]/home/components/composer/composer.tsx index 9189db61670..1c897d4d532 100644 --- a/apps/sim/app/o/[organizationId]/home/components/composer/composer.tsx +++ b/apps/sim/app/o/[organizationId]/home/components/composer/composer.tsx @@ -3,8 +3,12 @@ import { useRef } from 'react' import { Button, cn } from '@sim/emcn' import { ArrowUp } from '@sim/emcn/icons' +import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' +import { MicButton } from '@/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button' +import { MicrophonePermissionHelp } from '@/app/workspace/[workspaceId]/home/components/user-input/components/microphone-permission-help/microphone-permission-help' import { useAnimatedPlaceholder } from '@/hooks/use-animated-placeholder' import { useChatInputFocus } from '@/hooks/use-chat-input-focus' +import { useVoiceInput } from '@/hooks/use-voice-input' const SEND_BUTTON_BASE = 'size-[28px] rounded-full border-0 p-0 transition-colors' const SEND_BUTTON_ACTIVE = @@ -35,11 +39,23 @@ export function Composer({ onStop, }: ComposerProps) { const textareaRef = useRef(null) + const { organization } = useOrganizationContext() useChatInputFocus({ textareaRef }) + const voice = useVoiceInput({ + organizationId: organization.id, + getValue: () => value, + onChange, + }) const canSubmit = value.trim().length > 0 const animatedPlaceholder = useAnimatedPlaceholder(isInitialView) const placeholder = isInitialView ? animatedPlaceholder : 'Send message to Sim' + const submit = () => { + if (!canSubmit) return + voice.resetTranscript() + onSubmit() + } + return (
{ if (event.key === 'Enter' && !event.shiftKey && !event.nativeEvent.isComposing) { event.preventDefault() - onSubmit() + submit() } }} placeholder={placeholder} @@ -70,7 +86,14 @@ export function Composer({ />
-
+
+ {voice.isSupported && ( + + )} {isSending ? (
+
) } diff --git a/apps/sim/app/o/[organizationId]/search/search.test.tsx b/apps/sim/app/o/[organizationId]/search/search.test.tsx index 24ef1c21007..41db4871567 100644 --- a/apps/sim/app/o/[organizationId]/search/search.test.tsx +++ b/apps/sim/app/o/[organizationId]/search/search.test.tsx @@ -6,14 +6,21 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' import type { ResourceScope } from '@/lib/core/resource-scope' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import type { useSpeechToText } from '@/hooks/use-speech-to-text' const mocks = vi.hoisted(() => ({ search: vi.fn(), urlUpdate: vi.fn(), push: vi.fn(), + speech: vi.fn(), + toggleListening: vi.fn(), })) -vi.mock('next/navigation', () => ({ useRouter: () => ({ push: mocks.push }) })) +vi.mock('@/hooks/use-speech-to-text', () => ({ useSpeechToText: mocks.speech })) +vi.mock('next/navigation', () => ({ + useRouter: () => ({ push: mocks.push }), + usePathname: () => '/o/organization-a/search', +})) vi.mock('@/app/o/[organizationId]/providers/organization-provider', () => ({ useOrganizationContext: () => ({ organization: { id: 'organization-a', name: 'Acme' }, @@ -54,6 +61,21 @@ let container: HTMLDivElement beforeEach(() => { vi.clearAllMocks() vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.stubGlobal( + 'matchMedia', + vi.fn(() => ({ + matches: false, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + })) + ) + mocks.speech.mockReturnValue({ + isSupported: true, + isListening: false, + audioLevelsRef: { current: new Float32Array(5) }, + toggleListening: mocks.toggleListening, + resetTranscript: vi.fn(), + }) mocks.search.mockImplementation((_scope: ResourceScope, query: string) => { const result: WorkspaceKnowledgeSearchResult = { documentId: `document-${query}`, @@ -113,6 +135,30 @@ function expectVisibleQuery(query: string) { } describe('organization Search query navigation', () => { + it.each(['', '?q=Orion'])( + 'dictates into the draft without searching until submit (%s)', + async (params) => { + await render(params) + await editDraft('Find') + const searchCalls = mocks.search.mock.calls.length + const mic = container.querySelector('button[aria-label="Voice input"]')! + expect(mic.nextElementSibling?.getAttribute('aria-label')).toBe('Search') + await act(async () => mic.click()) + expect(mocks.toggleListening).toHaveBeenCalledOnce() + const speech = mocks.speech.mock.calls.at(-1)![0] + expect(speech.organizationId).toBe('organization-a') + await act(async () => speech.onTranscript('release')) + await act(async () => speech.onTranscript('release notes')) + expect(searchInput().value).toBe('Find release notes') + expect(mocks.search).toHaveBeenCalledTimes(searchCalls) + expect(mocks.urlUpdate).not.toHaveBeenCalled() + await act(async () => { + container.querySelector('button[aria-label="Search"]')!.click() + }) + expectVisibleQuery('Find release notes') + } + ) + it('replaces the field draft and results when the committed URL query changes without remounting the page', async () => { await render('?q=Orion') expectVisibleQuery('Orion') diff --git a/apps/sim/app/o/[organizationId]/search/search.tsx b/apps/sim/app/o/[organizationId]/search/search.tsx index 581fcee0c2e..7ab8ac1173d 100644 --- a/apps/sim/app/o/[organizationId]/search/search.tsx +++ b/apps/sim/app/o/[organizationId]/search/search.tsx @@ -17,10 +17,13 @@ import { organizationSearchUrlKeys, } from '@/app/o/[organizationId]/search/search-params' import { KnowledgeSearchResults } from '@/app/workspace/[workspaceId]/home/components/knowledge-search-results' +import { MicButton } from '@/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button' +import { MicrophonePermissionHelp } from '@/app/workspace/[workspaceId]/home/components/user-input/components/microphone-permission-help/microphone-permission-help' import { SIDEBAR_DIVIDER_PAD_ABOVE_CLASS, SIDEBAR_DIVIDER_PAD_BELOW_CLASS, } from '@/app/workspace/[workspaceId]/w/components/sidebar/constants' +import { useVoiceInput } from '@/hooks/use-voice-input' const SUBMIT_BUTTON_BASE = 'size-[28px] shrink-0 rounded-full border-0 p-0 transition-colors' const SUBMIT_BUTTON_ACTIVE = @@ -49,7 +52,13 @@ function SearchField({ docked = false, }: SearchFieldProps) { const inputRef = useRef(null) + const { organization } = useOrganizationContext() const [value, setValue] = useState(initialValue) + const voice = useVoiceInput({ + organizationId: organization.id, + getValue: () => value, + onChange: setValue, + }) const canSubmit = value.trim().length > 0 useEffect(() => { @@ -79,21 +88,34 @@ function SearchField({ aria-label='Search your sources' autoComplete='off' spellCheck={false} - className='h-full w-full bg-transparent font-body text-[14px] text-[var(--text-primary)] tracking-[-0.015em] outline-hidden placeholder:text-[var(--text-muted)] [&::-webkit-search-cancel-button]:hidden' + className='h-full min-w-0 flex-1 bg-transparent font-body text-[14px] text-[var(--text-primary)] tracking-[-0.015em] outline-hidden placeholder:text-[var(--text-muted)] [&::-webkit-search-cancel-button]:hidden' /> - + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx index 8b78f27fe03..2e2b6a51cae 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx @@ -5,7 +5,7 @@ import { Button, cn, Tooltip, usePrefersReducedMotion } from '@sim/emcn' import { Mic } from '@sim/emcn/icons' const WAVEFORM_BAR_COUNT = 5 -const WAVEFORM_MIN_HEIGHT = 3 +const WAVEFORM_MIN_HEIGHT = 1 const WAVEFORM_MAX_HEIGHT = 14 const WAVEFORM_CENTER = 9 const WAVEFORM_EASING = 0.24 @@ -71,7 +71,7 @@ function VoiceWaveform({ audioLevelsRef, isListening }: VoiceWaveformProps) { y2={WAVEFORM_CENTER + WAVEFORM_MIN_HEIGHT / 2} stroke='currentColor' strokeLinecap='round' - strokeWidth='1.7' + strokeWidth='1.5' /> ) })} @@ -108,6 +108,7 @@ export const MicButton = memo(function MicButton({ (function UserI const contextsEnabled = !canSearch || mode === 'build' const contextsEnabledRef = useRef(contextsEnabled) contextsEnabledRef.current = contextsEnabled - const [microphonePermissionHelpOpen, setMicrophonePermissionHelpOpen] = useState(false) const [initialValue] = useState(() => { if (defaultValue) return defaultValue @@ -332,14 +330,6 @@ const UserInputImpl = forwardRef(function UserI if (defaultValue) editorRef.current.setValue(defaultValue) }, [defaultValue]) - const sttPrefixRef = useRef('') - - function handleTranscript(text: string) { - const prefix = sttPrefixRef.current - const newVal = prefix ? `${prefix} ${text}` : text - editorRef.current.setValue(newVal) - } - function handleUsageLimitExceeded(message?: string, isMemberLimit?: boolean) { // A per-member cap can only be raised by an org admin, so don't offer Upgrade // (the member can't act on it) — the message already tells them to ask an admin. @@ -356,59 +346,21 @@ const UserInputImpl = forwardRef(function UserI ) } - function handleSpeechError(error: SpeechToTextError) { - if (error === 'microphone-blocked') { - const desktopBridge = getDesktopBridge() - if (desktopBridge) { - const { openMicrophoneSettings } = desktopBridge - toast.error( - 'Microphone access is blocked. Allow Sim to use the microphone in your system privacy settings.', - openMicrophoneSettings - ? { - action: { - label: 'Open Settings', - onClick: () => void openMicrophoneSettings(), - }, - } - : undefined - ) - } else { - toast.error('Microphone access is blocked. Allow it for this site and try again.', { - action: { - label: 'Show steps', - onClick: () => setMicrophonePermissionHelpOpen(true), - }, - }) - } - return - } - if (error === 'microphone-unavailable') { - toast.error('No microphone found. Connect one and try again.') - return - } - toast.error('Could not start voice input. Try again.') - } - const { audioLevelsRef, isListening, isSupported: isSttSupported, - toggleListening: rawToggle, + toggleListening, resetTranscript, - } = useSpeechToText({ - onTranscript: handleTranscript, - onUsageLimitExceeded: handleUsageLimitExceeded, - onError: handleSpeechError, + permissionHelpOpen, + setPermissionHelpOpen, + } = useVoiceInput({ workspaceId, + getValue: () => editorRef.current.getValue(), + onChange: (value) => editorRef.current.setValue(value), + onUsageLimitExceeded: handleUsageLimitExceeded, }) - const toggleListening = useCallback(() => { - if (!isListening) { - sttPrefixRef.current = editorRef.current.getValue() - } - rawToggle() - }, [isListening, rawToggle]) - const onSendQueuedHeadRef = useRef(onSendQueuedHead) onSendQueuedHeadRef.current = onSendQueuedHead const onEditQueuedTailRef = useRef(onEditQueuedTail) @@ -553,7 +505,6 @@ const UserInputImpl = forwardRef(function UserI /** Empties the text, chips, attachments, transcript, and the saved draft in one step. */ const clearComposer = useCallback(() => { editorRef.current.clear() - sttPrefixRef.current = '' if (draftSaveTimerRef.current !== null) { window.clearTimeout(draftSaveTimerRef.current) draftSaveTimerRef.current = null @@ -758,10 +709,7 @@ const UserInputImpl = forwardRef(function UserI {files.isDragging && } - + ) }) diff --git a/apps/sim/hooks/use-speech-to-text.ts b/apps/sim/hooks/use-speech-to-text.ts index f1a1d8986c8..cf7e3bf90fd 100644 --- a/apps/sim/hooks/use-speech-to-text.ts +++ b/apps/sim/hooks/use-speech-to-text.ts @@ -50,6 +50,8 @@ interface UseSpeechToTextProps { onError?: (error: SpeechToTextError) => void /** Attributes the voice-input cost to this workspace for per-member usage. */ workspaceId?: string + /** Attributes organization chat and search voice input to the organization. */ + organizationId?: string } interface UseSpeechToTextReturn { @@ -89,6 +91,7 @@ export function useSpeechToText({ onUsageLimitExceeded, onError, workspaceId, + organizationId, }: UseSpeechToTextProps): UseSpeechToTextReturn { const [isListening, setIsListening] = useState(false) /** @@ -108,6 +111,7 @@ export function useSpeechToText({ const onUsageLimitExceededRef = useRef(onUsageLimitExceeded) const onErrorRef = useRef(onError) const workspaceIdRef = useRef(workspaceId) + const organizationIdRef = useRef(organizationId) const mountedRef = useRef(true) const startingRef = useRef(false) @@ -128,6 +132,7 @@ export function useSpeechToText({ onUsageLimitExceededRef.current = onUsageLimitExceeded onErrorRef.current = onError workspaceIdRef.current = workspaceId + organizationIdRef.current = organizationId const flushAudioBuffer = useCallback(() => { const ws = wsRef.current @@ -213,7 +218,9 @@ export function useSpeechToText({ let tokenData: Awaited>> try { tokenData = await requestJson(speechTokenContract, { - body: workspaceIdRef.current ? { workspaceId: workspaceIdRef.current } : {}, + body: organizationIdRef.current + ? { organizationId: organizationIdRef.current } + : { workspaceId: workspaceIdRef.current }, }) } catch (err) { if (isApiClientError(err) && err.status === 402) { diff --git a/apps/sim/hooks/use-voice-input.test.tsx b/apps/sim/hooks/use-voice-input.test.tsx new file mode 100644 index 00000000000..f3b40a6528b --- /dev/null +++ b/apps/sim/hooks/use-voice-input.test.tsx @@ -0,0 +1,144 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { useSpeechToText } from '@/hooks/use-speech-to-text' + +const mocks = vi.hoisted(() => ({ speech: vi.fn() })) +vi.mock('@/hooks/use-speech-to-text', () => ({ useSpeechToText: mocks.speech })) + +import { useVoiceInput } from '@/hooks/use-voice-input' + +let root: Root +let container: HTMLDivElement +let value: string +let voice: ReturnType + +function Harness() { + voice = useVoiceInput({ + organizationId: 'organization-a', + getValue: () => value, + onChange: (next) => { + value = next + }, + }) + return null +} + +function transcript(text: string) { + act(() => mocks.speech.mock.calls.at(-1)![0].onTranscript(text)) +} + +beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + mocks.speech.mockReturnValue({ + isListening: false, + isSupported: true, + audioLevelsRef: { current: new Float32Array(5) }, + toggleListening: vi.fn(), + resetTranscript: vi.fn(), + }) + value = 'Find' + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root.render()) + act(() => voice.toggleListening()) +}) + +afterEach(() => { + act(() => root.unmount()) + container.remove() + vi.unstubAllGlobals() +}) + +describe('voice input manual edits', () => { + it('preserves edits made before the first transcript arrives', () => { + value = 'Summarize' + transcript('the report') + expect(value).toBe('Summarize the report') + }) + + it('preserves a changed prefix while updating partial speech', () => { + transcript('the repor') + value = 'Summarize the repor' + transcript('the report') + expect(value).toBe('Summarize the report') + transcript('the report from June') + expect(value).toBe('Summarize the report from June') + }) + + it.each(['Replacement draft', ''])( + 'preserves a replaced or cleared draft (%s)', + (replacement) => { + transcript('the report') + value = replacement + transcript('the report') + expect(value).toBe(replacement) + transcript('the report from June') + expect(value).toBe(replacement ? `${replacement} from June` : 'from June') + } + ) + + it('preserves corrections inside dictated text across later partial updates', () => { + transcript('the red report') + value = 'Find the blue report' + transcript('the red report from June') + expect(value).toBe('Find the blue report from June') + transcript('the red report from July') + expect(value).toBe('Find the blue report from July') + }) + + it('keeps manual suffixes after speech revisions and before new speech', () => { + transcript('the red report') + value += ' and notes' + transcript('the blue report') + expect(value).toBe('Find the blue report and notes') + transcript('the blue report from June') + expect(value).toBe('Find the blue report and notes from June') + }) + + it('prefers the manual correction when speech revises the same word', () => { + value = '' + act(() => voice.toggleListening()) + transcript('hel') + value = 'help' + transcript('hello') + expect(value).toBe('help') + transcript('hello again') + expect(value).toBe('help again') + }) + + it('keeps appended speech when another revision conflicts with a manual correction', () => { + transcript('the red report') + value = 'Find the blue report' + transcript('the green report from June') + expect(value).toBe('Find the blue report from June') + transcript('the green report from June and July') + expect(value).toBe('Find the blue report from June and July') + }) + + it('preserves separate manual edits around an independently revised word', () => { + transcript('the red report') + value = 'Summarize the red report and notes' + transcript('the green report from June') + expect(value).toBe('Summarize the green report and notes from June') + }) + + it('starts fresh after the draft and transcript are cleared on submit', () => { + transcript('the report') + value = '' + act(() => voice.resetTranscript()) + transcript('Next question') + expect(value).toBe('Next question') + }) + + it('continues dictation after a large draft is manually replaced', () => { + const longTranscript = Array.from({ length: 300 }, (_, index) => `word${index}`).join(' ') + transcript(longTranscript) + value = 'New draft' + transcript(`${longTranscript} next question`) + expect(value).toBe('New draft next question') + }) +}) diff --git a/apps/sim/hooks/use-voice-input.ts b/apps/sim/hooks/use-voice-input.ts new file mode 100644 index 00000000000..694490d45db --- /dev/null +++ b/apps/sim/hooks/use-voice-input.ts @@ -0,0 +1,160 @@ +'use client' + +import { useCallback, useRef, useState } from 'react' +import { toast } from '@sim/emcn' +import { diffWordsWithSpace } from 'diff' +import { getDesktopBridge } from '@/lib/desktop' +import { type SpeechToTextError, useSpeechToText } from '@/hooks/use-speech-to-text' + +interface UseVoiceInputProps { + workspaceId?: string + organizationId?: string + getValue: () => string + onChange: (value: string) => void + onUsageLimitExceeded?: (message?: string, isMemberLimit?: boolean) => void +} + +interface TextEdit { + from: number + to: number + insert: string +} + +function textChanges(before: string, after: string): TextEdit[] { + const parts = diffWordsWithSpace(before, after, { maxEditLength: 256 }) + if (!parts) return [{ from: 0, to: before.length, insert: after }] + const edits: TextEdit[] = [] + let position = 0 + let edit: TextEdit | undefined + for (const part of parts) { + if (part.added || part.removed) { + edit ??= { from: position, to: position, insert: '' } + if (part.added) edit.insert += part.value + if (part.removed) { + position += part.value.length + edit.to = position + } + } else { + if (edit) edits.push(edit) + edit = undefined + position += part.value.length + } + } + if (edit) edits.push(edit) + return edits +} + +/** Rebase independent speech updates around manual edits; the user's text wins conflicts. */ +function mergeTranscript(previous: string, next: string, current: string): string { + if (current === previous) return next + if (next === previous) return current + + const speechEdits = textChanges(previous, next) + const manualEdits = textChanges(previous, current) + let result = current + for (const speech of speechEdits.reverse()) { + let offset = 0 + let conflict = false + for (const edit of manualEdits) { + if (speech.to <= edit.from && speech.from < edit.from) continue + if (speech.from >= edit.to && (speech.from > edit.from || /^\s/.test(speech.insert))) { + offset += edit.insert.length - (edit.to - edit.from) + } else { + conflict = true + break + } + } + if (!conflict) { + result = + result.slice(0, speech.from + offset) + speech.insert + result.slice(speech.to + offset) + } + } + return current ? result : result.trimStart() +} + +/** Shares draft appending and microphone recovery across chat and search inputs. */ +export function useVoiceInput({ + workspaceId, + organizationId, + getValue, + onChange, + onUsageLimitExceeded = (message) => toast.error(message || 'You are out of credits.'), +}: UseVoiceInputProps) { + const prefixRef = useRef('') + const previousTranscriptValueRef = useRef('') + const getValueRef = useRef(getValue) + getValueRef.current = getValue + const [permissionHelpOpen, setPermissionHelpOpen] = useState(false) + + function handleSpeechError(error: SpeechToTextError) { + if (error === 'microphone-blocked') { + const desktopBridge = getDesktopBridge() + if (desktopBridge) { + const { openMicrophoneSettings } = desktopBridge + toast.error( + 'Microphone access is blocked. Allow Sim to use the microphone in your system privacy settings.', + openMicrophoneSettings + ? { + action: { + label: 'Open Settings', + onClick: () => void openMicrophoneSettings(), + }, + } + : undefined + ) + } else { + toast.error('Microphone access is blocked. Allow it for this site and try again.', { + action: { + label: 'Show steps', + onClick: () => setPermissionHelpOpen(true), + }, + }) + } + return + } + if (error === 'microphone-unavailable') { + toast.error('No microphone found. Connect one and try again.') + return + } + toast.error('Could not start voice input. Try again.') + } + + const { + toggleListening: rawToggle, + resetTranscript: rawReset, + ...speech + } = useSpeechToText({ + workspaceId, + organizationId, + onTranscript: (text) => { + const next = prefixRef.current ? `${prefixRef.current} ${text}` : text + const value = mergeTranscript(previousTranscriptValueRef.current, next, getValueRef.current()) + previousTranscriptValueRef.current = next + onChange(value) + }, + onUsageLimitExceeded, + onError: handleSpeechError, + }) + + const toggleListening = useCallback(() => { + if (!speech.isListening) { + prefixRef.current = getValueRef.current() + previousTranscriptValueRef.current = prefixRef.current + } + rawToggle() + }, [speech.isListening, rawToggle]) + + const resetTranscript = useCallback(() => { + prefixRef.current = '' + previousTranscriptValueRef.current = '' + rawReset() + }, [rawReset]) + + return { + ...speech, + toggleListening, + resetTranscript, + permissionHelpOpen, + setPermissionHelpOpen, + } +} diff --git a/apps/sim/lib/api/contracts/media/speech.ts b/apps/sim/lib/api/contracts/media/speech.ts index 79c2918ad6b..cfde9a08a64 100644 --- a/apps/sim/lib/api/contracts/media/speech.ts +++ b/apps/sim/lib/api/contracts/media/speech.ts @@ -1,16 +1,14 @@ import { z } from 'zod' +import { resourceOwnerSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' -export const speechTokenBodySchema = z - .object({ - /** Workspace the session user is recording in. */ - workspaceId: z.string().optional(), - }) - .passthrough() +export const speechTokenBodySchema = resourceOwnerSchema +export type SpeechTokenBody = z.input export const speechTokenResponseSchema = z.object({ token: z.string(), }) +export type SpeechTokenResponse = z.output export const speechTokenContract = defineRouteContract({ method: 'POST', diff --git a/apps/sim/lib/speech/application/create-token.ts b/apps/sim/lib/speech/application/create-token.ts new file mode 100644 index 00000000000..a9f09027186 --- /dev/null +++ b/apps/sim/lib/speech/application/create-token.ts @@ -0,0 +1,155 @@ +import { createHash } from 'node:crypto' +import type { Principal } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { z } from 'zod' +import { + type BillingAttributionSnapshot, + checkAttributedUsageLimits, + resolveBillingAttribution, + resolveOrganizationBillingAttribution, + toBillingContext, +} from '@/lib/billing/core/billing-attribution' +import { recordUsage } from '@/lib/billing/core/usage-log' +import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application/authorized-workspace-use-case' +import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' +import { requireAllowedWorkspacePrincipal } from '@/lib/core/application/workspace-authorization' +import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' +import { env } from '@/lib/core/config/env' +import { getCostMultiplier } from '@/lib/core/config/env-flags' +import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' +import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' + +const logger = createLogger('SpeechToken') +const ELEVENLABS_TOKEN_URL = 'https://api.elevenlabs.io/v1/single-use-token/realtime_scribe' +const VOICE_SESSION_COST_PER_MIN = 0.008 +const VOICE_SESSION_MAX_MINUTES = 3 +const providerTokenSchema = z.object({ token: z.string().min(1) }) +const providerErrorSchema = z.object({ + detail: z.string().optional(), + message: z.string().optional(), +}) + +/** + * permission-group-exempt: Voice dictation is an input aid shared across product surfaces. + */ +export const speechTokenOperation = defineWorkspaceOperation({ + id: 'speech.token.create', + minimumRole: 'read', + workspaceApiKey: 'deny', + principalKinds: ['session'], + capability: 'none', +}) + +/** + * permission-group-exempt: Voice dictation is an input aid shared across product surfaces. + */ +const organizationSpeechTokenOperation = defineOrganizationOperation({ + id: 'speech.token.create', + minimumRole: 'member', + principalKinds: ['session'], + capability: 'none', +}) + +export class SpeechTokenError extends Error { + constructor( + readonly reason: 'usage_limit' | 'unconfigured' | 'provider_failed', + message: string, + readonly scope?: 'actor' | 'payer' | 'member' + ) { + super(message) + this.name = 'SpeechTokenError' + } +} + +async function issueSpeechToken( + actorUserId: string, + billingAttribution: BillingAttributionSnapshot +) { + /** Admission remains bounded by the per-user token bucket; billing has no reservation primitive. */ + const usageCheck = await checkAttributedUsageLimits(billingAttribution) + if (usageCheck.isExceeded) { + throw new SpeechTokenError( + 'usage_limit', + usageCheck.message || 'Usage limit exceeded. Please upgrade your plan to continue.', + usageCheck.scope + ) + } + + const apiKey = env.ELEVENLABS_API_KEY + if (!apiKey?.trim()) { + throw new SpeechTokenError('unconfigured', 'Speech-to-text service is not configured') + } + + const response = await fetch(ELEVENLABS_TOKEN_URL, { + method: 'POST', + headers: { 'xi-api-key': apiKey }, + }) + if (!response.ok) { + const error = providerErrorSchema.safeParse(await response.json().catch(() => ({}))) + const message = + (error.success && (error.data.detail || error.data.message)) || + `Token request failed (${response.status})` + logger.error('ElevenLabs token request failed', { status: response.status, message }) + throw new SpeechTokenError('provider_failed', message) + } + + const { token } = providerTokenSchema.parse(await response.json()) + try { + await recordUsage({ + userId: actorUserId, + ...(billingAttribution.workspaceId ? { workspaceId: billingAttribution.workspaceId } : {}), + ...toBillingContext(billingAttribution), + entries: [ + { + category: 'fixed', + source: 'voice-input', + description: `Voice input session (${VOICE_SESSION_MAX_MINUTES} min)`, + cost: VOICE_SESSION_COST_PER_MIN * VOICE_SESSION_MAX_MINUTES * getCostMultiplier(), + sourceReference: `voice-input:${createHash('sha256').update(token).digest('hex')}`, + }, + ], + }) + await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) + } catch (error) { + logger.warn('Failed to record voice input usage, continuing:', error) + } + return { token } +} + +const createWorkspaceSpeechToken = defineAuthorizedWorkspaceUseCase({ + operation: speechTokenOperation, + resolveContext: ({ input }: { input: { workspaceId: string } }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: {}, + async execute({ principal, context }) { + const attribution = await resolveBillingAttribution({ + actorUserId: principal.userId, + workspaceId: context.workspaceId, + }) + return issueSpeechToken(principal.userId, attribution) + }, +}) + +/** Issues a metered token under the caller's current workspace or organization membership. */ +export const createSpeechToken = { + operation: speechTokenOperation, + async execute({ principal, input }: { principal: Principal; input: ResourceOwner }) { + requireAllowedWorkspacePrincipal(principal, speechTokenOperation) + const scope = resourceScopeFromOwner(input) + if (scope.kind === 'workspace') { + return createWorkspaceSpeechToken.execute({ principal, input: scope }) + } + const context = await authorizeOrganizationOperation( + principal, + organizationSpeechTokenOperation, + scope + ) + const attribution = await resolveOrganizationBillingAttribution({ + actorUserId: context.userId, + organizationId: context.organizationId, + }) + return issueSpeechToken(context.userId, attribution) + }, +} diff --git a/apps/sim/package.json b/apps/sim/package.json index 35600921107..ec558e036bc 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -175,6 +175,7 @@ "csv-parse": "7.0.2", "date-fns": "4.1.0", "decimal.js": "10.6.0", + "diff": "8.0.4", "docx-preview": "^0.3.7", "docx": "^9.6.1", "drizzle-orm": "^0.45.2", diff --git a/bun.lock b/bun.lock index 3b63e4d5571..848b03f4155 100644 --- a/bun.lock +++ b/bun.lock @@ -290,6 +290,7 @@ "csv-parse": "7.0.2", "date-fns": "4.1.0", "decimal.js": "10.6.0", + "diff": "8.0.4", "docx": "^9.6.1", "docx-preview": "^0.3.7", "drizzle-orm": "^0.45.2",