diff --git a/.changeset/nightly-auto-approvals.md b/.changeset/nightly-auto-approvals.md new file mode 100644 index 0000000000..4050abee03 --- /dev/null +++ b/.changeset/nightly-auto-approvals.md @@ -0,0 +1,7 @@ +--- +'@roomote/web': patch +--- + + + +Move Auto tool approvals to Nightly Experiments and keep saved customer opt-ins disabled. diff --git a/apps/web/src/components/settings/IntegrationToolAutoApprovalsNightlySetting.client.test.tsx b/apps/web/src/components/settings/IntegrationToolAutoApprovalsNightlySetting.client.test.tsx new file mode 100644 index 0000000000..4dd6be7c9c --- /dev/null +++ b/apps/web/src/components/settings/IntegrationToolAutoApprovalsNightlySetting.client.test.tsx @@ -0,0 +1,26 @@ +import { fireEvent, render, screen } from '@testing-library/react'; + +const setEnabled = vi.fn(); +vi.mock('@/hooks/useDeploymentExperiments', () => ({ + useDeploymentExperiment: () => ({ + enabled: false, + isLoading: false, + isUpdating: false, + setEnabled, + }), +})); + +import { IntegrationToolAutoApprovalsNightlySetting } from './IntegrationToolAutoApprovalsNightlySetting'; + +it('offers the Auto switch on the nightly page and updates its flag', () => { + render(); + + expect(screen.getByText('Auto tool approvals')).toBeInTheDocument(); + expect( + screen.getByText(/Jev is required for Auto to make decisions/), + ).toBeInTheDocument(); + fireEvent.click( + screen.getByRole('switch', { name: 'Toggle Auto tool approvals' }), + ); + expect(setEnabled).toHaveBeenCalledWith(true); +}); diff --git a/apps/web/src/components/settings/IntegrationToolAutoApprovalsNightlySetting.tsx b/apps/web/src/components/settings/IntegrationToolAutoApprovalsNightlySetting.tsx new file mode 100644 index 0000000000..934d5372dc --- /dev/null +++ b/apps/web/src/components/settings/IntegrationToolAutoApprovalsNightlySetting.tsx @@ -0,0 +1,32 @@ +'use client'; + +import { ShieldQuestion, Switch } from '@/components/system'; +import { useDeploymentExperiment } from '@/hooks/useDeploymentExperiments'; + +import { Section } from './Section'; + +export function IntegrationToolAutoApprovalsNightlySetting() { + const { enabled, isLoading, isUpdating, setEnabled } = + useDeploymentExperiment( + 'integrationToolAutoApprovals', + 'Failed to update Auto tool approvals.', + ); + + return ( +
+
+ +

+ Expose the Auto-approval decisions card in Settings → Agent Guidance. + Jev is required for Auto to make decisions, and saved per-tool choices + continue to take priority. +

+
+
+ ); +} diff --git a/apps/web/src/components/settings/IntegrationToolAutoModeSetting.client.test.tsx b/apps/web/src/components/settings/IntegrationToolAutoModeSetting.client.test.tsx index 4745aeab9c..6ceb896c15 100644 --- a/apps/web/src/components/settings/IntegrationToolAutoModeSetting.client.test.tsx +++ b/apps/web/src/components/settings/IntegrationToolAutoModeSetting.client.test.tsx @@ -146,4 +146,22 @@ describe('IntegrationToolAutoModeSetting', () => { policy: 'Existing guidance', }); }); + + it('only allows Auto-on when the resolved model is Jev', () => { + state.settings = { + mode: 'off', + policy: '', + model: { kind: 'helper', model: 'openai/gpt-test' }, + }; + const { rerender } = render(); + expect( + screen.getByRole('switch', { name: 'Enable auto-approval' }), + ).toBeDisabled(); + + state.settings = { mode: 'off', policy: '', model: { kind: 'judgment' } }; + rerender(); + expect( + screen.getByRole('switch', { name: 'Enable auto-approval' }), + ).toBeEnabled(); + }); }); diff --git a/apps/web/src/components/settings/McpToolManagementDialog.client.test.tsx b/apps/web/src/components/settings/McpToolManagementDialog.client.test.tsx index d8a8927165..0780ed36a8 100644 --- a/apps/web/src/components/settings/McpToolManagementDialog.client.test.tsx +++ b/apps/web/src/components/settings/McpToolManagementDialog.client.test.tsx @@ -134,6 +134,27 @@ describe('McpToolManagementDialog tool approvals', () => { } }); + it('keeps explicit manual approvals available while Auto is off', () => { + renderDialog(); + + const search = within( + screen.getByRole('group', { + name: 'Approval mode for web_search_exa', + }), + ); + fireEvent.click(search.getByRole('button', { name: 'Always ask' })); + fireEvent.click(search.getByRole('button', { name: 'Disable' })); + + expect(state.setModeCalls).toEqual([ + { integrationId: 'exa', toolName: 'web_search_exa', mode: 'ask' }, + { integrationId: 'exa', toolName: 'web_search_exa', mode: 'reject' }, + ]); + expect(search.getByRole('button', { name: 'Always allow' })).toBeEnabled(); + expect( + search.queryByRole('button', { name: 'Auto' }), + ).not.toBeInTheDocument(); + }); + it('does not fire the admin-only policy query while the dialog is closed or for non-admin viewers', () => { state.autoEnabled = true; state.policiesQueryEnabled = undefined; diff --git a/apps/web/src/components/settings/pages/NightlyExperimentsPage.client.test.tsx b/apps/web/src/components/settings/pages/NightlyExperimentsPage.client.test.tsx index 225bdadb53..abf730a4ec 100644 --- a/apps/web/src/components/settings/pages/NightlyExperimentsPage.client.test.tsx +++ b/apps/web/src/components/settings/pages/NightlyExperimentsPage.client.test.tsx @@ -48,6 +48,15 @@ vi.mock('@/components/settings/SettingsShell', () => ({ ), })); +vi.mock( + '@/components/settings/IntegrationToolAutoApprovalsNightlySetting', + () => ({ + IntegrationToolAutoApprovalsNightlySetting: () => ( +
Auto tool approvals setting
+ ), + }), +); + import { NightlyExperimentsPage } from './NightlyExperimentsPage'; describe('NightlyExperimentsPage', () => { @@ -83,6 +92,7 @@ describe('NightlyExperimentsPage', () => { ), ).toBeInTheDocument(); expect(screen.getByText('Dizzy')).toBeInTheDocument(); + expect(screen.getByText('Auto tool approvals setting')).toBeInTheDocument(); expect( screen.getByText( 'Continuously spin the Roomote logo mark in the collapsed sidebar and mobile header.', diff --git a/apps/web/src/components/settings/pages/NightlyExperimentsPage.tsx b/apps/web/src/components/settings/pages/NightlyExperimentsPage.tsx index 3295f84d45..bf2ab2b9c7 100644 --- a/apps/web/src/components/settings/pages/NightlyExperimentsPage.tsx +++ b/apps/web/src/components/settings/pages/NightlyExperimentsPage.tsx @@ -5,6 +5,7 @@ import { ExperimentSettingsControls, getExperimentSettingIds, } from '@/components/settings/experiment-settings-controls'; +import { IntegrationToolAutoApprovalsNightlySetting } from '@/components/settings/IntegrationToolAutoApprovalsNightlySetting'; import { EmptyState, Moon, RetryableLoadError } from '@/components/system'; import { useDeploymentExperiments } from '@/hooks/useDeploymentExperiments'; @@ -38,7 +39,11 @@ export function NightlyExperimentsPage() { description="There are no internal nightly controls available on this deployment." /> ) : ( - + <> + {/* Auto's runtime-aware switch belongs only on this gated page. */} + + + )} ); diff --git a/apps/web/src/hooks/useDeploymentExperiments.client.test.tsx b/apps/web/src/hooks/useDeploymentExperiments.client.test.tsx index 98b2f895b6..9b967dd1c3 100644 --- a/apps/web/src/hooks/useDeploymentExperiments.client.test.tsx +++ b/apps/web/src/hooks/useDeploymentExperiments.client.test.tsx @@ -65,6 +65,9 @@ vi.mock('@/trpc/client', () => ({ dizzyEnabled: { queryKey: () => ['dizzy-enabled'], }, + integrationToolAutoApprovalsEnabled: { + queryKey: () => ['integration-tool-auto-approvals-enabled'], + }, set: { mutationOptions: (options: unknown) => { mocks.mutationRoute = 'internal-nightly'; @@ -148,6 +151,27 @@ describe('useDeploymentExperiments', () => { }); }); + it('refreshes Auto runtime consumers after changing its nightly experiment', async () => { + renderHook(() => + useDeploymentExperiments('Save failed', 'internal-nightly'), + ); + + await mutationOptions.onSettled!( + undefined as never, + undefined as never, + { id: 'integrationToolAutoApprovals', enabled: true } as never, + undefined as never, + undefined as never, + ); + + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: ['nightly-experiments'], + }); + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: ['integration-tool-auto-approvals-enabled'], + }); + }); + it('optimistically updates and rolls back only the changed deployment flag', async () => { const previous = getDeploymentExperimentValues(undefined); mocks.getQueryData.mockReturnValue(previous); diff --git a/apps/web/src/hooks/useDeploymentExperiments.ts b/apps/web/src/hooks/useDeploymentExperiments.ts index a952b76ed4..9d0d860622 100644 --- a/apps/web/src/hooks/useDeploymentExperiments.ts +++ b/apps/web/src/hooks/useDeploymentExperiments.ts @@ -100,6 +100,12 @@ export function useDeploymentExperiments( queryKey: trpc.nightlyExperiments.dizzyEnabled.queryKey(), }); } + if (isNightly && variables.id === 'integrationToolAutoApprovals') { + void queryClient.invalidateQueries({ + queryKey: + trpc.nightlyExperiments.integrationToolAutoApprovalsEnabled.queryKey(), + }); + } }, }; diff --git a/apps/web/src/hooks/useIntegrationToolAutoApprovalsExperiment.client.test.tsx b/apps/web/src/hooks/useIntegrationToolAutoApprovalsExperiment.client.test.tsx new file mode 100644 index 0000000000..a21b5c441d --- /dev/null +++ b/apps/web/src/hooks/useIntegrationToolAutoApprovalsExperiment.client.test.tsx @@ -0,0 +1,69 @@ +import { renderHook } from '@testing-library/react'; + +const { queryState, state } = vi.hoisted(() => ({ + queryState: { data: false as boolean | undefined, isPending: false }, + state: { + queryOptions: vi.fn(), + nightlyExperimentsEnabled: false, + }, +})); + +vi.mock('@tanstack/react-query', () => ({ + useQuery: (options: unknown) => { + state.queryOptions(options); + return queryState; + }, +})); + +vi.mock('@/hooks/useUser', () => ({ + useAuthorizedUser: () => ({ + nightlyExperimentsEnabled: state.nightlyExperimentsEnabled, + }), +})); + +vi.mock('@/trpc/client', () => ({ + useTRPC: () => ({ + nightlyExperiments: { + integrationToolAutoApprovalsEnabled: { + queryOptions: (_input: undefined, options: unknown) => ({ options }), + }, + }, + }), +})); + +import { useIntegrationToolAutoApprovalsExperiment } from './useIntegrationToolAutoApprovalsExperiment'; + +describe('useIntegrationToolAutoApprovalsExperiment', () => { + beforeEach(() => { + state.nightlyExperimentsEnabled = false; + queryState.data = false; + queryState.isPending = false; + }); + + it('keeps Auto off and skips runtime reads outside nightly deployments', () => { + queryState.data = true; + + const { result } = renderHook(() => + useIntegrationToolAutoApprovalsExperiment(), + ); + + expect(result.current).toEqual({ enabled: false, isLoading: false }); + expect(state.queryOptions).toHaveBeenCalledWith( + expect.objectContaining({ options: { enabled: false } }), + ); + }); + + it('uses the nightly runtime value for all signed-in viewers on opted-in deployments', () => { + state.nightlyExperimentsEnabled = true; + queryState.data = true; + + const { result } = renderHook(() => + useIntegrationToolAutoApprovalsExperiment(), + ); + + expect(result.current.enabled).toBe(true); + expect(state.queryOptions).toHaveBeenCalledWith( + expect.objectContaining({ options: { enabled: true } }), + ); + }); +}); diff --git a/apps/web/src/hooks/useIntegrationToolAutoApprovalsExperiment.ts b/apps/web/src/hooks/useIntegrationToolAutoApprovalsExperiment.ts index 8df92e316d..e19c72d287 100644 --- a/apps/web/src/hooks/useIntegrationToolAutoApprovalsExperiment.ts +++ b/apps/web/src/hooks/useIntegrationToolAutoApprovalsExperiment.ts @@ -1,14 +1,28 @@ 'use client'; -import { useDeploymentExperiment } from './useDeploymentExperiments'; +import { useQuery } from '@tanstack/react-query'; + +import { useAuthorizedUser } from '@/hooks/useUser'; +import { useTRPC } from '@/trpc/client'; /** - * Auto tool approvals are experimental; per-tool approvals are not. While - * this is off, a tool nobody has made a choice about simply runs. + * Auto tool approvals are an internal nightly experiment; per-tool approvals + * are not. The runtime value is readable by members only on opted-in internal + * deployments, so ordinary deployments never query the admin-only settings + * procedure. */ export function useIntegrationToolAutoApprovalsExperiment() { - return useDeploymentExperiment( - 'integrationToolAutoApprovals', - 'Failed to update Auto tool approvals.', + const { nightlyExperimentsEnabled } = useAuthorizedUser(); + const trpc = useTRPC(); + const query = useQuery( + trpc.nightlyExperiments.integrationToolAutoApprovalsEnabled.queryOptions( + undefined, + { enabled: nightlyExperimentsEnabled === true }, + ), ); + + return { + enabled: nightlyExperimentsEnabled === true && query.data === true, + isLoading: nightlyExperimentsEnabled === true && query.isPending, + }; } diff --git a/apps/web/src/trpc/commands/deployment-experiments/index.test.ts b/apps/web/src/trpc/commands/deployment-experiments/index.test.ts index 1fc24daccd..a9c905481e 100644 --- a/apps/web/src/trpc/commands/deployment-experiments/index.test.ts +++ b/apps/web/src/trpc/commands/deployment-experiments/index.test.ts @@ -114,7 +114,10 @@ describe('deployment experiment commands', () => { await expect( getNightlyExperimentsCommand(auth(admin.id, true, true)), - ).resolves.toEqual({ dizzy: false }); + ).resolves.toEqual({ + integrationToolAutoApprovals: false, + dizzy: false, + }); await expect( getDeploymentExperimentsCommand(auth(admin.id, true, true)), ).resolves.toHaveProperty('privateSessions'); diff --git a/apps/web/src/trpc/commands/deployment-experiments/index.ts b/apps/web/src/trpc/commands/deployment-experiments/index.ts index c5f0d906ae..c46959671f 100644 --- a/apps/web/src/trpc/commands/deployment-experiments/index.ts +++ b/apps/web/src/trpc/commands/deployment-experiments/index.ts @@ -65,6 +65,21 @@ export async function getDizzyExperimentEnabledCommand( return (await getDeploymentExperiments()).dizzy; } +/** + * Auto is deployment-wide runtime behavior, so signed-in users on an + * explicitly opted-in internal deployment may read its enabled state. Only + * admins can manage the switch or see the other nightly experiment values. + */ +export async function getIntegrationToolAutoApprovalsEnabledCommand( + auth: UserAuthSuccess, +): Promise { + if (auth.nightlyExperimentsEnabled !== true) { + throw new Error('Unauthorized'); + } + + return (await getDeploymentExperiments()).integrationToolAutoApprovals; +} + export async function setNightlyExperimentCommand( auth: UserAuthSuccess, input: { id: DeploymentExperimentId; enabled: boolean }, diff --git a/apps/web/src/trpc/commands/deployment-experiments/integration-tool-auto-approvals.test.ts b/apps/web/src/trpc/commands/deployment-experiments/integration-tool-auto-approvals.test.ts new file mode 100644 index 0000000000..dbd40795af --- /dev/null +++ b/apps/web/src/trpc/commands/deployment-experiments/integration-tool-auto-approvals.test.ts @@ -0,0 +1,67 @@ +import type { UserAuthSuccess } from '@/types'; + +import { + getDeploymentExperimentsCommand, + getIntegrationToolAutoApprovalsEnabledCommand, + getNightlyExperimentsCommand, + setDeploymentExperimentCommand, + setNightlyExperimentCommand, +} from './index'; + +function auth( + userId: string, + isAdmin: boolean, + nightlyExperimentsEnabled = false, +): UserAuthSuccess { + return { userId, isAdmin, nightlyExperimentsEnabled } as UserAuthSuccess; +} + +describe('Auto tool approvals nightly experiment', () => { + const admin = auth('auto-admin', true, true); + const member = auth('auto-member', false, true); + + it('keeps Auto out of customer-preview reads and writes', async () => { + await expect( + getDeploymentExperimentsCommand(admin), + ).resolves.not.toHaveProperty('integrationToolAutoApprovals'); + await expect( + setDeploymentExperimentCommand(admin, { + id: 'integrationToolAutoApprovals', + enabled: true, + }), + ).rejects.toThrow('Unauthorized'); + }); + + it('defaults off in Nightly and exposes the runtime value to opted-in members', async () => { + await setNightlyExperimentCommand(admin, { + id: 'integrationToolAutoApprovals', + enabled: false, + }); + await expect(getNightlyExperimentsCommand(admin)).resolves.toMatchObject({ + integrationToolAutoApprovals: false, + }); + await expect( + getIntegrationToolAutoApprovalsEnabledCommand(member), + ).resolves.toBe(false); + await expect( + getIntegrationToolAutoApprovalsEnabledCommand( + auth('customer-member', false, false), + ), + ).rejects.toThrow('Unauthorized'); + + try { + await setNightlyExperimentCommand(admin, { + id: 'integrationToolAutoApprovals', + enabled: true, + }); + await expect( + getIntegrationToolAutoApprovalsEnabledCommand(member), + ).resolves.toBe(true); + } finally { + await setNightlyExperimentCommand(admin, { + id: 'integrationToolAutoApprovals', + enabled: false, + }); + } + }); +}); diff --git a/apps/web/src/trpc/commands/integration-tool-policies/index.test.ts b/apps/web/src/trpc/commands/integration-tool-policies/index.test.ts new file mode 100644 index 0000000000..fa4658c38b --- /dev/null +++ b/apps/web/src/trpc/commands/integration-tool-policies/index.test.ts @@ -0,0 +1,27 @@ +import type { UserAuthSuccess } from '@/types'; + +import { + getIntegrationToolAutoSettingsCommand, + setIntegrationToolAutoSettingsCommand, +} from './index'; + +describe('Auto settings nightly access', () => { + const customerAdmin = { + userId: 'admin', + isAdmin: true, + nightlyExperimentsEnabled: false, + } as UserAuthSuccess; + + it('keeps old Auto settings unavailable on deployments without nightly opt-in', async () => { + await expect( + getIntegrationToolAutoSettingsCommand(customerAdmin), + ).rejects.toThrow('Auto tool approvals are not enabled.'); + + await expect( + setIntegrationToolAutoSettingsCommand(customerAdmin, { + mode: 'on', + policy: '', + }), + ).rejects.toThrow('Auto tool approvals are not enabled.'); + }); +}); diff --git a/apps/web/src/trpc/commands/integration-tool-policies/index.ts b/apps/web/src/trpc/commands/integration-tool-policies/index.ts index 4f3dd2d21d..091d8c2766 100644 --- a/apps/web/src/trpc/commands/integration-tool-policies/index.ts +++ b/apps/web/src/trpc/commands/integration-tool-policies/index.ts @@ -68,7 +68,8 @@ export async function setIntegrationToolPoliciesCommand( return listIntegrationToolPolicies(); } -const autoApprovalsEnabled = () => +const autoApprovalsEnabled = (auth: UserAuthSuccess) => + auth.nightlyExperimentsEnabled === true && isDeploymentExperimentEnabled('integrationToolAutoApprovals'); /** @@ -211,6 +212,12 @@ export async function getIntegrationToolAutoSettingsCommand( auth: UserAuthSuccess, ) { assertAdmin(auth); + if (!(await autoApprovalsEnabled(auth))) { + throw new TRPCError({ + code: 'NOT_FOUND', + message: 'Auto tool approvals are not enabled.', + }); + } const [settings, model] = await Promise.all([ getIntegrationToolAutoSettings(), resolveDecisionModel(AUTO_DECISION_REQUIREMENTS).catch(() => null), @@ -231,7 +238,7 @@ export async function setIntegrationToolAutoSettingsCommand( input: IntegrationToolAutoSettings, ) { assertAdmin(auth); - if (!(await autoApprovalsEnabled())) { + if (!(await autoApprovalsEnabled(auth))) { throw new TRPCError({ code: 'NOT_FOUND', message: 'Auto tool approvals are not enabled.', diff --git a/apps/web/src/trpc/routers/_app.ts b/apps/web/src/trpc/routers/_app.ts index d382a714b4..0580d610aa 100644 --- a/apps/web/src/trpc/routers/_app.ts +++ b/apps/web/src/trpc/routers/_app.ts @@ -230,6 +230,7 @@ import { import { getDizzyExperimentEnabledCommand, getDeploymentExperimentsCommand, + getIntegrationToolAutoApprovalsEnabledCommand, getNightlyExperimentsCommand, setDeploymentExperimentCommand, setNightlyExperimentCommand, @@ -3681,6 +3682,10 @@ export const appRouter = createRouter({ dizzyEnabled: protectedProcedure.query(({ ctx: { auth } }) => getDizzyExperimentEnabledCommand(auth), ), + integrationToolAutoApprovalsEnabled: protectedProcedure.query( + ({ ctx: { auth } }) => + getIntegrationToolAutoApprovalsEnabledCommand(auth), + ), get: protectedProcedure.query(({ ctx: { auth } }) => getNightlyExperimentsCommand(auth), ), diff --git a/apps/web/src/trpc/routers/nightly-experiments.test.ts b/apps/web/src/trpc/routers/nightly-experiments.test.ts index 731aa357fa..cdb147de22 100644 --- a/apps/web/src/trpc/routers/nightly-experiments.test.ts +++ b/apps/web/src/trpc/routers/nightly-experiments.test.ts @@ -23,6 +23,9 @@ describe('nightly experiment API access', () => { await expect(caller.nightlyExperiments.dizzyEnabled()).rejects.toThrow( 'Unauthorized', ); + await expect( + caller.nightlyExperiments.integrationToolAutoApprovalsEnabled(), + ).rejects.toThrow('Unauthorized'); }); it('keeps management routes admin-only on opted-in deployments', async () => { diff --git a/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts b/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts index 8f5d867b85..a90274d517 100644 --- a/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts +++ b/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts @@ -5,6 +5,10 @@ const mocks = vi.hoisted(() => ({ recordShadow: vi.fn(async () => undefined), settings: vi.fn(async () => ({ mode: 'off', policy: '' })), experiment: vi.fn(async () => true), + nightlyExperimentsEnabled: true as boolean | string, + isEnvFlagEnabled: vi.fn( + (value: unknown) => value === true || value === 'true' || value === '1', + ), })); vi.mock('../typesafe-judgment', () => ({ @@ -17,6 +21,14 @@ vi.mock('@roomote/db/server', async () => ({ recordIntegrationToolAutoEvaluation: mocks.record, recordIntegrationToolShadowEvaluation: mocks.recordShadow, })); +vi.mock('@roomote/env', () => ({ + Env: { + get R_NIGHTLY_EXPERIMENTS_ENABLED() { + return mocks.nightlyExperimentsEnabled; + }, + }, + isEnvFlagEnabled: mocks.isEnvFlagEnabled, +})); import { evaluateIntegrationToolAutoDecision, @@ -62,6 +74,8 @@ beforeEach(() => { mocks.settings.mockResolvedValue({ mode: 'off', policy: '' }); mocks.resolveModel.mockResolvedValue({ kind: 'judgment' }); mocks.experiment.mockResolvedValue(true); + mocks.nightlyExperimentsEnabled = true; + mocks.isEnvFlagEnabled.mockClear(); }); describe('recommendFromAutoAnswers', () => { @@ -432,6 +446,33 @@ describe('resolveIntegrationToolAutoState', () => { expect(mocks.experiment).toHaveBeenCalledWith( 'integrationToolAutoApprovals', ); + expect(mocks.resolveModel).toHaveBeenCalledWith({ + excludeRoomoteModel: true, + }); + }); + + it('keeps saved customer opt-ins off unless the deployment explicitly opts into nightly experiments', async () => { + mocks.nightlyExperimentsEnabled = false; + mocks.settings.mockResolvedValue({ mode: 'on', policy: 'Old guidance' }); + + await expect(resolveIntegrationToolAutoState()).resolves.toEqual({ + mode: 'off', + settings: { mode: 'on', policy: 'Old guidance' }, + model: null, + }); + expect(mocks.resolveModel).not.toHaveBeenCalled(); + }); + + it('treats a string false nightly flag as disabled when env validation is skipped', async () => { + mocks.nightlyExperimentsEnabled = 'false'; + mocks.settings.mockResolvedValue({ mode: 'on', policy: 'Old guidance' }); + + await expect(resolveIntegrationToolAutoState()).resolves.toMatchObject({ + mode: 'off', + model: null, + }); + expect(mocks.isEnvFlagEnabled).toHaveBeenCalledWith('false'); + expect(mocks.resolveModel).not.toHaveBeenCalled(); }); it('shadows while off with a hosted model, so its judgment can be reviewed', async () => { diff --git a/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts b/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts index 6b98b36da0..66464a6e52 100644 --- a/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts +++ b/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts @@ -367,19 +367,24 @@ export type IntegrationToolAutoState = { }; export async function resolveIntegrationToolAutoState(): Promise { - const [enabled, settings, model] = await Promise.all([ + // Some callers import this module only for the Jev requirements; defer Env + // initialization until the Auto state is actually resolved. + const [enabled, settings, nightlyExperimentsEnabled] = await Promise.all([ isDeploymentExperimentEnabled('integrationToolAutoApprovals'), getIntegrationToolAutoSettings(), - resolveDecisionModel(AUTO_DECISION_REQUIREMENTS).catch(() => null), + import('@roomote/env').then(({ Env, isEnvFlagEnabled }) => + isEnvFlagEnabled(Env.R_NIGHTLY_EXPERIMENTS_ENABLED), + ), ]); + if (!nightlyExperimentsEnabled || !enabled) { + return { mode: 'off', settings, model: null }; + } + + const model = await resolveDecisionModel(AUTO_DECISION_REQUIREMENTS).catch( + () => null, + ); const hosted = model?.kind === 'judgment'; - const mode = !enabled - ? 'off' - : settings.mode === 'on' - ? 'on' - : hosted - ? 'shadow' - : 'off'; + const mode = settings.mode === 'on' ? 'on' : hosted ? 'shadow' : 'off'; return { mode, settings, model: model?.kind ?? null }; } diff --git a/packages/db/src/lib/__tests__/integration-tool-approvals.test.ts b/packages/db/src/lib/__tests__/integration-tool-approvals.test.ts index bfeb9303bc..ae3e29c63f 100644 --- a/packages/db/src/lib/__tests__/integration-tool-approvals.test.ts +++ b/packages/db/src/lib/__tests__/integration-tool-approvals.test.ts @@ -1,9 +1,11 @@ import { db, + deploymentSettings, eq, integrationToolApprovalRequests, integrationToolAutoEvaluations, sessionFactory, + sql, sessions, taskFactory, userFactory, @@ -495,6 +497,41 @@ describe('Auto mode', () => { await setIntegrationToolAutoSettings({ mode: 'off', policy: '' }); }); + it('does not revive a saved Auto-on mode when the nightly experiment is first enabled', async () => { + await db + .insert(deploymentSettings) + .values({ + id: 'default', + metadata: { + integration_tool_auto: { + mode: 'on', + policy: 'Reads are fine. Never send messages.', + }, + }, + setupCompletedAt: null, + }) + .onConflictDoUpdate({ + target: deploymentSettings.id, + set: { + metadata: sql`coalesce(${deploymentSettings.metadata}, '{}'::jsonb) - 'integration_tool_auto_nightly' || ${JSON.stringify( + { + integration_tool_auto: { + mode: 'on', + policy: 'Reads are fine. Never send messages.', + }, + }, + )}::jsonb`, + }, + }); + + await expect(getIntegrationToolAutoSettings()).resolves.toEqual({ + mode: 'off', + policy: 'Reads are fine. Never send messages.', + }); + + await setIntegrationToolAutoSettings({ mode: 'off', policy: '' }); + }); + it('stores Always allow as a choice of its own, unlike the default', async () => { const userId = await user(); const integrationId = `always-${Date.now()}`; diff --git a/packages/db/src/lib/integration-tool-auto-settings.ts b/packages/db/src/lib/integration-tool-auto-settings.ts index c24addc000..ccaea3e7b9 100644 --- a/packages/db/src/lib/integration-tool-auto-settings.ts +++ b/packages/db/src/lib/integration-tool-auto-settings.ts @@ -9,12 +9,15 @@ import { type DatabaseOrTransaction, db } from '../db'; import { deploymentSettings } from '../schema'; const DEFAULT_DEPLOYMENT_ID = 'default'; -const METADATA_KEY = 'integration_tool_auto'; +const METADATA_KEY = 'integration_tool_auto_nightly'; +const LEGACY_METADATA_KEY = 'integration_tool_auto'; /** * Deployment-wide Auto mode for tool approvals, kept with the other - * deployment settings. Off by default: default tools run as they always - * have until an admin turns Auto on. + * deployment settings. The nightly key intentionally does not inherit the + * former customer-preview mode, so turning on the internal experiment cannot + * silently revive an old Auto opt-in. Preserve its saved guidance, but require + * a fresh choice to turn Auto on. */ const DEFAULTS: IntegrationToolAutoSettings = { mode: 'off', policy: '' }; @@ -25,10 +28,18 @@ export async function getIntegrationToolAutoSettings( where: eq(deploymentSettings.id, DEFAULT_DEPLOYMENT_ID), columns: { metadata: true }, }); - const parsed = integrationToolAutoSettingsSchema.safeParse( - (deployment?.metadata as Record | null)?.[METADATA_KEY], + const metadata = deployment?.metadata as Record | null; + const current = integrationToolAutoSettingsSchema.safeParse( + metadata?.[METADATA_KEY], ); - return parsed.success ? parsed.data : DEFAULTS; + if (current.success) return current.data; + + const legacy = integrationToolAutoSettingsSchema.safeParse( + metadata?.[LEGACY_METADATA_KEY], + ); + return legacy.success + ? { mode: 'off', policy: legacy.data.policy } + : DEFAULTS; } export async function setIntegrationToolAutoSettings( diff --git a/packages/feature-flags/src/__tests__/integration-tool-auto-approvals.test.ts b/packages/feature-flags/src/__tests__/integration-tool-auto-approvals.test.ts new file mode 100644 index 0000000000..5b795d44b1 --- /dev/null +++ b/packages/feature-flags/src/__tests__/integration-tool-auto-approvals.test.ts @@ -0,0 +1,35 @@ +import { + getBooleanMetadataDescriptorByKey, + getDeploymentExperimentAudience, + getDeploymentExperimentIdsForAudience, + getDeploymentExperimentValues, +} from '../index'; + +describe('Auto tool approvals nightly experiment metadata', () => { + it('ignores legacy customer-preview opt-ins and defaults the new nightly key off', () => { + expect( + getDeploymentExperimentValues({ + integration_tool_auto_approvals_experiment_enabled: true, + }).integrationToolAutoApprovals, + ).toBe(false); + expect( + getDeploymentExperimentValues({ + integration_tool_auto_approvals_nightly_experiment_enabled: true, + }).integrationToolAutoApprovals, + ).toBe(true); + expect( + getBooleanMetadataDescriptorByKey( + 'integration_tool_auto_approvals_experiment_enabled', + ).kind, + ).toBe('legacy'); + }); + + it('classifies Auto tool approvals as an internal-nightly experiment', () => { + expect( + getDeploymentExperimentAudience('integrationToolAutoApprovals'), + ).toBe('internal-nightly'); + expect(getDeploymentExperimentIdsForAudience('internal-nightly')).toContain( + 'integrationToolAutoApprovals', + ); + }); +}); diff --git a/packages/feature-flags/src/__tests__/metadata-descriptions.test.ts b/packages/feature-flags/src/__tests__/metadata-descriptions.test.ts index 9833032362..bd2cfe7844 100644 --- a/packages/feature-flags/src/__tests__/metadata-descriptions.test.ts +++ b/packages/feature-flags/src/__tests__/metadata-descriptions.test.ts @@ -71,6 +71,7 @@ describe('metadata descriptions', () => { undefined, ); expect(getDeploymentExperimentIdsForAudience('internal-nightly')).toEqual([ + 'integrationToolAutoApprovals', 'dizzy', ]); }); diff --git a/packages/feature-flags/src/config.ts b/packages/feature-flags/src/config.ts index e7758a794b..c9574b0d9a 100644 --- a/packages/feature-flags/src/config.ts +++ b/packages/feature-flags/src/config.ts @@ -27,7 +27,7 @@ export const DEPLOYMENT_EXPERIMENT_AUDIENCE = { privateSessions: 'customer-preview', sessionTaskCommunicationTriage: 'customer-preview', browserNotifications: 'customer-preview', - integrationToolAutoApprovals: 'customer-preview', + integrationToolAutoApprovals: 'internal-nightly', dizzy: 'internal-nightly', } as const satisfies Record< DeploymentExperimentId, @@ -37,8 +37,9 @@ export const DEPLOYMENT_EXPERIMENT_AUDIENCE = { export const DEPLOYMENT_EXPERIMENT_METADATA_KEYS = { privateSessions: 'private_sessions_experiment_enabled', browserNotifications: 'browser_notifications_experiment_enabled', + // A new key deliberately leaves the former customer-preview opt-ins dormant. integrationToolAutoApprovals: - 'integration_tool_auto_approvals_experiment_enabled', + 'integration_tool_auto_approvals_nightly_experiment_enabled', sessionTaskCommunicationTriage: 'session_task_communication_triage_experiment_enabled', dizzy: 'dizzy_experiment_enabled', @@ -85,7 +86,7 @@ export const DEPLOYMENT_METADATA_BOOLEAN_CONFIG: Record< kind: 'deployment-control', group: null, description: - 'Show the Auto-approval decisions card in Settings → Agent Guidance for admins to turn on.', + 'Show the Auto-approval decisions card in Settings → Agent Guidance for admins on internal nightly deployments.', }, [DEPLOYMENT_EXPERIMENT_METADATA_KEYS.sessionTaskCommunicationTriage]: { kind: 'deployment-control',