diff --git a/.changeset/nightly-automation-criteria.md b/.changeset/nightly-automation-criteria.md new file mode 100644 index 0000000000..4dfb08e593 --- /dev/null +++ b/.changeset/nightly-automation-criteria.md @@ -0,0 +1,7 @@ +--- +'@roomote/web': patch +--- + + + +Move custom automation launch-criteria controls to the internal Nightly experiments page. diff --git a/apps/api/src/handlers/custom-automations/__tests__/custom-automations-routes.test.ts b/apps/api/src/handlers/custom-automations/__tests__/custom-automations-routes.test.ts index ff87b0c6ad..940fc55ab7 100644 --- a/apps/api/src/handlers/custom-automations/__tests__/custom-automations-routes.test.ts +++ b/apps/api/src/handlers/custom-automations/__tests__/custom-automations-routes.test.ts @@ -8,6 +8,7 @@ import type { } from '@roomote/types'; import { ALL_REPOSITORIES, + CUSTOM_AUTOMATION_LAUNCH_CRITERIA_MAX_LENGTH, CUSTOM_AUTOMATION_PROMPT_MAX_LENGTH, FAST_EXECUTION, MANAGE_CUSTOM_AUTOMATIONS_TOOL, @@ -27,8 +28,10 @@ const { mockCreateCustomAutomation, mockUpdateCustomAutomation, mockGetCustomAutomationById, + mockListCustomAutomationConditionRuns, mockListCustomAutomations, mockGetDeploymentTaskModelOptions, + mockIsDeploymentExperimentEnabled, mockDeleteCustomAutomation, mockListConnectedCommunicationProviders, mockCanStartAgentMailConversationWithUser, @@ -43,8 +46,10 @@ const { mockCreateCustomAutomation: vi.fn(), mockUpdateCustomAutomation: vi.fn(), mockGetCustomAutomationById: vi.fn(), + mockListCustomAutomationConditionRuns: vi.fn(), mockListCustomAutomations: vi.fn(), mockGetDeploymentTaskModelOptions: vi.fn(), + mockIsDeploymentExperimentEnabled: vi.fn(), mockDeleteCustomAutomation: vi.fn(), mockListConnectedCommunicationProviders: vi.fn(), mockCanStartAgentMailConversationWithUser: vi.fn(), @@ -65,8 +70,10 @@ vi.mock('@roomote/db/server', () => ({ updateCustomAutomation: mockUpdateCustomAutomation, deleteCustomAutomation: mockDeleteCustomAutomation, getCustomAutomationById: mockGetCustomAutomationById, + listCustomAutomationConditionRuns: mockListCustomAutomationConditionRuns, listCustomAutomations: mockListCustomAutomations, getDeploymentTaskModelOptions: mockGetDeploymentTaskModelOptions, + isDeploymentExperimentEnabled: mockIsDeploymentExperimentEnabled, })); vi.mock('@roomote/sdk/server', () => ({ @@ -160,7 +167,7 @@ function postCreate( }); } -function registerApiHostedTool(auth: McpAuth) { +async function registerApiHostedTool(auth: McpAuth) { let handler: | ((params: ManageCustomAutomationsInput) => Promise) | undefined; @@ -174,7 +181,7 @@ function registerApiHostedTool(auth: McpAuth) { }, ); - registerRoomoteCustomAutomationsTool( + await registerRoomoteCustomAutomationsTool( { registerTool } as unknown as McpServer, auth, ); @@ -202,6 +209,20 @@ describe('custom-automations MCP routes', () => { models: ENABLED_MODELS, defaultModelId: 'openai/gpt-5.6-luna', }); + mockIsDeploymentExperimentEnabled.mockResolvedValue(true); + mockListCustomAutomationConditionRuns.mockResolvedValue([]); + }); + + it('returns the launch-criteria experiment state to the worker MCP server', async () => { + const { app } = createApp(); + mockIsDeploymentExperimentEnabled.mockResolvedValue(true); + + const response = await app.request('/custom-automations/experiment'); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ + launchCriteriaEnabled: true, + }); }); describe('API-hosted Roomote MCP tool', () => { @@ -213,7 +234,7 @@ describe('custom-automations MCP routes', () => { principal: 'user', version: 1, }; - const { handler, registerTool } = registerApiHostedTool({ + const { handler, registerTool } = await registerApiHostedTool({ userId: 'admin-1', authContext, }); @@ -241,13 +262,34 @@ describe('custom-automations MCP routes', () => { }); }); + it('omits launch-condition fields from the tool schema while disabled', async () => { + mockIsDeploymentExperimentEnabled.mockResolvedValue(false); + const { registerTool } = await registerApiHostedTool({ + userId: 'admin-1', + authContext: { + userId: 'admin-1', + tokenType: 'auth', + version: 1, + }, + }); + + const registeredConfig = registerTool.mock.calls[0]?.[1] as { + description: string; + inputSchema: Record; + }; + expect(registeredConfig.inputSchema).not.toHaveProperty('launchCriteria'); + expect(registeredConfig.inputSchema).not.toHaveProperty('runWhen'); + expect(registeredConfig.description).not.toContain('launchCriteria'); + expect(registeredConfig.description).not.toContain('runWhen'); + }); + it("lists the owner's Email identities when the tool scopes list_destinations to an automation", async () => { const authContext: AuthTokenContext = { userId: 'admin-1', tokenType: 'auth', version: 1, }; - const { handler } = registerApiHostedTool({ + const { handler } = await registerApiHostedTool({ userId: 'admin-1', authContext, }); @@ -291,7 +333,7 @@ describe('custom-automations MCP routes', () => { tokenType: 'auth', version: 1, }; - const { handler } = registerApiHostedTool({ + const { handler } = await registerApiHostedTool({ userId: 'admin-1', authContext, }); @@ -349,7 +391,7 @@ describe('custom-automations MCP routes', () => { tokenType: 'auth', version: 1, }; - const { handler } = registerApiHostedTool({ + const { handler } = await registerApiHostedTool({ userId: 'admin-1', authContext, }); @@ -375,6 +417,7 @@ describe('custom-automations MCP routes', () => { name: 'Nightly report', prompt: 'Inspect this stored prompt.', }, + conditionRuns: [], }); }); @@ -384,7 +427,7 @@ describe('custom-automations MCP routes', () => { tokenType: 'auth', version: 1, }; - const { handler } = registerApiHostedTool({ + const { handler } = await registerApiHostedTool({ userId: 'admin-1', authContext, }); @@ -426,7 +469,7 @@ describe('custom-automations MCP routes', () => { tokenType: 'auth', version: 1, }; - const { handler } = registerApiHostedTool({ + const { handler } = await registerApiHostedTool({ userId: 'admin-1', authContext, }); @@ -450,7 +493,7 @@ describe('custom-automations MCP routes', () => { tokenType: 'auth', version: 1, }; - const { handler } = registerApiHostedTool({ + const { handler } = await registerApiHostedTool({ userId: 'member-1', authContext, }); @@ -629,15 +672,48 @@ describe('custom-automations MCP routes', () => { expect(mockRunCustomAutomationNow).not.toHaveBeenCalled(); }); - it('returns a bounded stored prompt record by automation ID', async () => { + it('returns the saved condition and recent condition runs by automation ID', async () => { const { app } = createApp(); + const runWhen = { + all: [ + { + id: 'new_regression', + ask: 'Does `report` describe a new regression?', + type: 'yes_no', + criteria: { true: 'New regression.', false: 'No new regression.' }, + min: 0.75, + }, + ], + onUncertain: 'skip', + }; mockGetCustomAutomationById.mockResolvedValue({ id: 'automation-1', name: 'Nightly report', prompt: 'Inspect this stored prompt.', + launchCriteria: 'Only investigate new regressions.', + runWhen, enabled: true, lastError: 'previous failure', }); + mockListCustomAutomationConditionRuns.mockResolvedValue([ + { + id: 'result-1', + createdAt: new Date('2026-09-23T12:00:00.000Z'), + launchCriteriaSnapshot: { + launchCriteria: 'Only investigate new regressions.', + runWhen, + }, + launchCriteriaAnswers: { + criteriaMet: { type: 'noul', noul: 0.1 }, + runWhen: { new_regression: { type: 'noul', noul: 0.9 } }, + }, + launchCriteriaOutcome: { + launchCriteria: 'skipped', + runWhen: 'passed', + }, + content: 'No qualifying regression found.', + }, + ]); const res = await app.request('/custom-automations/automation-1'); @@ -647,7 +723,26 @@ describe('custom-automations MCP routes', () => { id: 'automation-1', name: 'Nightly report', prompt: 'Inspect this stored prompt.', + launchCriteria: 'Only investigate new regressions.', + runWhen, }, + conditionRuns: [ + { + id: 'result-1', + createdAt: '2026-09-23T12:00:00.000Z', + outcome: 'passed', + runWhen, + answers: { + new_regression: { type: 'noul', noul: 0.9 }, + }, + launchCriteria: 'Only investigate new regressions.', + launchCriteriaOutcome: 'skipped', + launchCriteriaAnswers: { + criteriaMet: { type: 'noul', noul: 0.1 }, + }, + findingsExcerpt: 'No qualifying regression found.', + }, + ], }); }); @@ -664,6 +759,84 @@ describe('custom-automations MCP routes', () => { }); describe('POST / (create)', () => { + it.each(['launchCriteria', 'runWhen'] as const)( + 'rejects %s while the experiment is disabled', + async (field) => { + mockIsDeploymentExperimentEnabled.mockResolvedValue(false); + const { app } = createApp(); + const response = await postCreate( + app, + createBody({ + [field]: + field === 'launchCriteria' + ? 'Only investigate new regressions.' + : { all: [] }, + }), + ); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toEqual({ + error: 'Custom automation launch criteria are not enabled.', + }); + expect(mockCreateCustomAutomation).not.toHaveBeenCalled(); + }, + ); + + it('accepts and persists declarative runWhen conditions', async () => { + const { app } = createApp(); + const runWhen = { + all: [ + { + id: 'new_regression', + ask: 'Does `report` describe a new regression?', + type: 'yes_no', + criteria: { true: 'New regression.', false: 'No new regression.' }, + min: 0.75, + }, + ], + onUncertain: 'skip', + }; + mockCreateCustomAutomation.mockResolvedValue({ id: 'automation-1' }); + + const res = await postCreate(app, createBody({ runWhen })); + + expect(res.status).toBe(201); + expect(mockCreateCustomAutomation).toHaveBeenCalledWith( + expect.objectContaining({ runWhen }), + ); + }); + + it('accepts bounded plain-language launch criteria', async () => { + const { app } = createApp(); + const launchCriteria = 'Only investigate new regressions.'; + mockCreateCustomAutomation.mockResolvedValue({ + id: 'automation-1', + launchCriteria, + }); + + const res = await postCreate(app, createBody({ launchCriteria })); + + expect(res.status).toBe(201); + expect(mockCreateCustomAutomation).toHaveBeenCalledWith( + expect.objectContaining({ launchCriteria }), + ); + }); + + it('rejects launch criteria beyond the shared limit', async () => { + const { app } = createApp(); + const res = await postCreate( + app, + createBody({ + launchCriteria: 'x'.repeat( + CUSTOM_AUTOMATION_LAUNCH_CRITERIA_MAX_LENGTH + 1, + ), + }), + ); + + expect(res.status).toBe(400); + expect(mockCreateCustomAutomation).not.toHaveBeenCalled(); + }); + it('accepts a prompt at the shared 16,000-character limit', async () => { const { app } = createApp(); const prompt = 'x'.repeat(CUSTOM_AUTOMATION_PROMPT_MAX_LENGTH); @@ -1159,6 +1332,61 @@ describe('custom-automations MCP routes', () => { target: {}, }; + it('updates declarative runWhen without requiring prompt changes', async () => { + const { app } = createApp(); + const runWhen = { + all: [ + { + id: 'new_regression', + ask: 'Does `report` describe a new regression?', + type: 'yes_no', + criteria: { true: 'New regression.', false: 'No new regression.' }, + min: 0.75, + }, + ], + onUncertain: 'skip', + }; + mockGetCustomAutomationById.mockResolvedValue(existing); + mockUpdateCustomAutomation.mockResolvedValue({ id: 'automation-1' }); + + const res = await app.request('/custom-automations/automation-1', { + method: 'PATCH', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ runWhen }), + }); + + expect(res.status).toBe(200); + expect(mockUpdateCustomAutomation).toHaveBeenCalledWith( + 'automation-1', + expect.objectContaining({ runWhen }), + ); + }); + + it.each(['launchCriteria', 'runWhen'] as const)( + 'rejects %s updates while the experiment is disabled', + async (field) => { + mockIsDeploymentExperimentEnabled.mockResolvedValue(false); + const { app } = createApp(); + const response = await app.request('/custom-automations/automation-1', { + method: 'PATCH', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + [field]: + field === 'launchCriteria' + ? 'Only investigate new regressions.' + : { all: [] }, + }), + }); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toEqual({ + error: 'Custom automation launch criteria are not enabled.', + }); + expect(mockGetCustomAutomationById).not.toHaveBeenCalled(); + expect(mockUpdateCustomAutomation).not.toHaveBeenCalled(); + }, + ); + it('accepts a prompt at the shared 16,000-character limit', async () => { const { app } = createApp(); mockGetCustomAutomationById.mockResolvedValue(existing); diff --git a/apps/api/src/handlers/custom-automations/index.ts b/apps/api/src/handlers/custom-automations/index.ts index 72bf8d51d7..3634c9a88c 100644 --- a/apps/api/src/handlers/custom-automations/index.ts +++ b/apps/api/src/handlers/custom-automations/index.ts @@ -10,6 +10,8 @@ import { eq, getDeploymentTaskModelOptions, getCustomAutomationById, + isDeploymentExperimentEnabled, + listCustomAutomationConditionRuns, isNull, listCustomAutomations, updateCustomAutomation, @@ -30,7 +32,9 @@ import { FAST_EXECUTION, REASONING_EFFORT_VALUES, AUTOMATION_RESULT_PRIORITIES, + CUSTOM_AUTOMATION_LAUNCH_CRITERIA_MAX_LENGTH, CUSTOM_AUTOMATION_PROMPT_MAX_LENGTH, + customAutomationRunWhenSchema, getAutomationTargetEmailIdentityId, getAutomationTargetKind, type BackgroundAutomationProvider, @@ -78,6 +82,13 @@ const writeSchema = z.object({ .optional(), targetMode: z.enum(['channel', 'direct_message']).optional(), targetChannelId: z.string().trim().min(1).max(160).optional(), + runWhen: customAutomationRunWhenSchema.nullable().optional(), + launchCriteria: z + .string() + .trim() + .max(CUSTOM_AUTOMATION_LAUNCH_CRITERIA_MAX_LENGTH) + .nullable() + .optional(), }); const updateSchema = z.object({ @@ -100,6 +111,13 @@ const updateSchema = z.object({ .optional(), targetMode: z.enum(['channel', 'direct_message']).optional(), targetChannelId: z.string().trim().min(1).max(160).optional(), + runWhen: customAutomationRunWhenSchema.nullable().optional(), + launchCriteria: z + .string() + .trim() + .max(CUSTOM_AUTOMATION_LAUNCH_CRITERIA_MAX_LENGTH) + .nullable() + .optional(), }); const UNIQUE_VIOLATION_CODE = '23505'; @@ -107,6 +125,15 @@ const NAME_UNIQUE_INDEX = 'custom_automations_name_unique_idx'; export const DUPLICATE_AUTOMATION_NAME_ERROR = 'A custom automation with this name already exists.'; +function hasLaunchCriteriaFields(value: unknown): boolean { + return ( + value !== null && + typeof value === 'object' && + !Array.isArray(value) && + ('launchCriteria' in value || 'runWhen' in value) + ); +} + /** * Whether the error (or anything in its cause chain — drizzle wraps the * driver error in a DrizzleQueryError) is the Postgres unique violation for @@ -397,6 +424,14 @@ customAutomationsRouter.get('/', async (c) => }), ); +customAutomationsRouter.get('/experiment', async (c) => + c.json({ + launchCriteriaEnabled: await isDeploymentExperimentEnabled( + 'automationLaunchCriteria', + ), + }), +); + customAutomationsRouter.get('/models', async (c) => c.json(await getDeploymentTaskModelOptions()), ); @@ -436,12 +471,36 @@ customAutomationsRouter.get('/:id', async (c) => { if (!automation || !canManage(c, automation)) { return c.json({ error: 'Custom automation was not found.' }, 404); } + const conditionRuns = await listCustomAutomationConditionRuns(automation.id); return c.json({ automation: { id: automation.id, name: automation.name, prompt: automation.prompt, + launchCriteria: automation.launchCriteria, + runWhen: automation.runWhen, }, + conditionRuns: conditionRuns.map((run) => { + const outcomes = run.launchCriteriaOutcome; + const launchCriteriaOutcome = outcomes?.launchCriteria ?? null; + const runWhenOutcome = outcomes?.runWhen ?? null; + const answers = run.launchCriteriaAnswers; + return { + id: run.id, + createdAt: run.createdAt, + outcome: runWhenOutcome ?? launchCriteriaOutcome, + runWhen: run.launchCriteriaSnapshot?.runWhen ?? null, + answers: answers?.runWhen ?? null, + ...(launchCriteriaOutcome || runWhenOutcome + ? { findingsExcerpt: run.content.slice(0, 2_000) } + : { reportExcerpt: run.content.slice(0, 2_000) }), + launchCriteria: run.launchCriteriaSnapshot?.launchCriteria ?? null, + launchCriteriaOutcome, + launchCriteriaAnswers: answers?.criteriaMet + ? { criteriaMet: answers.criteriaMet } + : null, + }; + }), }); }); @@ -465,7 +524,17 @@ customAutomationsRouter.post('/resolve-schedule', async (c) => { }); customAutomationsRouter.post('/', async (c) => { - const parsed = writeSchema.safeParse(await c.req.json()); + const body: unknown = await c.req.json(); + if ( + hasLaunchCriteriaFields(body) && + !(await isDeploymentExperimentEnabled('automationLaunchCriteria')) + ) { + return c.json( + { error: 'Custom automation launch criteria are not enabled.' }, + 400, + ); + } + const parsed = writeSchema.safeParse(body); if (!parsed.success) return c.json({ error: parsed.error.message }, 400); try { await assertEnabledModel(parsed.data.model, parsed.data.reasoningEffort); @@ -501,6 +570,12 @@ customAutomationsRouter.post('/', async (c) => { reasoningEffort: parsed.data.reasoningEffort ?? null, environmentId: parsed.data.environmentId, target: buildTarget(parsed.data, actorId(c)), + ...(parsed.data.launchCriteria !== undefined + ? { launchCriteria: parsed.data.launchCriteria } + : {}), + ...(parsed.data.runWhen !== undefined + ? { runWhen: parsed.data.runWhen } + : {}), createdByUserId: actorId(c), }); void captureActivationCustomAutomationChanged( @@ -522,7 +597,17 @@ customAutomationsRouter.post('/', async (c) => { }); customAutomationsRouter.patch('/:id', async (c) => { - const parsed = updateSchema.safeParse(await c.req.json()); + const body: unknown = await c.req.json(); + if ( + hasLaunchCriteriaFields(body) && + !(await isDeploymentExperimentEnabled('automationLaunchCriteria')) + ) { + return c.json( + { error: 'Custom automation launch criteria are not enabled.' }, + 400, + ); + } + const parsed = updateSchema.safeParse(body); if (!parsed.success) return c.json({ error: parsed.error.message }, 400); const existing = await getCustomAutomationById(c.req.param('id')); if (!existing || !canManage(c, existing)) { @@ -642,6 +727,12 @@ customAutomationsRouter.patch('/:id', async (c) => { existing.createdByUserId ?? actorId(c), ) : existingTarget, + ...(parsed.data.launchCriteria !== undefined + ? { launchCriteria: parsed.data.launchCriteria } + : {}), + ...(parsed.data.runWhen !== undefined + ? { runWhen: parsed.data.runWhen } + : {}), }); return c.json({ automation: toApiAutomation(automation), diff --git a/apps/api/src/handlers/mcp/roomote-custom-automations-tool.ts b/apps/api/src/handlers/mcp/roomote-custom-automations-tool.ts index 0b8bf0c4b1..2ad0945c8a 100644 --- a/apps/api/src/handlers/mcp/roomote-custom-automations-tool.ts +++ b/apps/api/src/handlers/mcp/roomote-custom-automations-tool.ts @@ -1,11 +1,13 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { - MANAGE_CUSTOM_AUTOMATIONS_TOOL, buildManageCustomAutomationsRequest, compactManageCustomAutomationsResult, + getManageCustomAutomationsTool, type ManageCustomAutomationsInput, } from '@roomote/types'; +import { isDeploymentExperimentEnabled } from '@roomote/db/server'; + import { customAutomationsRouter } from '../custom-automations'; import { invokeInProcessApi, @@ -17,6 +19,7 @@ import type { McpAuth } from './middleware'; async function invokeManageCustomAutomations( auth: McpAuth, params: ManageCustomAutomationsInput, + launchCriteriaEnabled: boolean, ) { const built = buildManageCustomAutomationsRequest(params); if (!built.ok) { @@ -42,22 +45,35 @@ async function invokeManageCustomAutomations( payload: compactManageCustomAutomationsResult( params.action, result.payload, + { includeLaunchCriteria: launchCriteriaEnabled }, ), }); } -export function registerRoomoteCustomAutomationsTool( +export async function registerRoomoteCustomAutomationsTool( server: McpServer, auth: McpAuth, -): void { +): Promise { + let launchCriteriaEnabled = false; + try { + launchCriteriaEnabled = await isDeploymentExperimentEnabled( + 'automationLaunchCriteria', + ); + } catch (error) { + console.warn( + `[MCP] Could not read custom automation launch-criteria experiment; hiding its fields: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const tool = getManageCustomAutomationsTool(launchCriteriaEnabled); server.registerTool( - MANAGE_CUSTOM_AUTOMATIONS_TOOL.name, + tool.name, { - title: MANAGE_CUSTOM_AUTOMATIONS_TOOL.title, - description: MANAGE_CUSTOM_AUTOMATIONS_TOOL.description, - inputSchema: MANAGE_CUSTOM_AUTOMATIONS_TOOL.inputSchema, - annotations: MANAGE_CUSTOM_AUTOMATIONS_TOOL.annotations, + title: tool.title, + description: tool.description, + inputSchema: tool.inputSchema, + annotations: tool.annotations, }, - (params) => invokeManageCustomAutomations(auth, params), + (params: ManageCustomAutomationsInput) => + invokeManageCustomAutomations(auth, params, launchCriteriaEnabled), ); } diff --git a/apps/api/src/handlers/mcp/roomote.ts b/apps/api/src/handlers/mcp/roomote.ts index 33df31d4ab..d2a3fbedc7 100644 --- a/apps/api/src/handlers/mcp/roomote.ts +++ b/apps/api/src/handlers/mcp/roomote.ts @@ -397,7 +397,7 @@ function createRoomoteTransport() { }); } -function createRoomoteMcpServer( +async function createRoomoteMcpServer( auth: McpAuthContext, actingUserId: string | null, toolAuth: McpAuth, @@ -413,7 +413,7 @@ function createRoomoteMcpServer( registerRoomoteCommunicationTools(server, actingUserId); } } - registerRoomoteCustomAutomationsTool(server, toolAuth); + await registerRoomoteCustomAutomationsTool(server, toolAuth); registerRoomoteCustomSkillsTool(server, toolAuth); registerRoomoteArtifactTool(server, toolAuth); registerRoomotePublicUrlFetchTool(server); @@ -591,7 +591,7 @@ function createRoomoteMcpRouter(options: { } : rawAuth, }; - const server = createRoomoteMcpServer( + const server = await createRoomoteMcpServer( auth, actingUserId, toolAuth, diff --git a/apps/web/src/components/settings/AutomationLaunchCriteriaExperimentalSetting.client.test.tsx b/apps/web/src/components/settings/AutomationLaunchCriteriaExperimentalSetting.client.test.tsx new file mode 100644 index 0000000000..81a7f971ea --- /dev/null +++ b/apps/web/src/components/settings/AutomationLaunchCriteriaExperimentalSetting.client.test.tsx @@ -0,0 +1,56 @@ +import { fireEvent, render, screen } from '@testing-library/react'; + +const state = vi.hoisted(() => ({ + enabled: false, + isLoading: false, + isUpdating: false, + setEnabled: vi.fn(), +})); + +vi.mock('@/hooks/useAutomationLaunchCriteriaExperiment', () => ({ + useAutomationLaunchCriteriaExperiment: () => state, +})); + +import { AutomationLaunchCriteriaExperimentalSetting } from './AutomationLaunchCriteriaExperimentalSetting'; + +describe('AutomationLaunchCriteriaExperimentalSetting', () => { + beforeEach(() => { + vi.clearAllMocks(); + state.enabled = false; + state.isLoading = false; + state.isUpdating = false; + }); + + it('shows the default-off setting and persists an explicit opt-in', () => { + render(); + + const toggle = screen.getByRole('switch', { + name: 'Toggle custom automation launch criteria', + }); + expect(toggle).not.toBeChecked(); + fireEvent.click(toggle); + expect(state.setEnabled).toHaveBeenCalledWith(true); + }); + + it('disables the setting while it is loading or saving', () => { + state.isUpdating = true; + const { rerender } = render( + , + ); + + expect( + screen.getByRole('switch', { + name: 'Toggle custom automation launch criteria', + }), + ).toBeDisabled(); + + state.isUpdating = false; + state.isLoading = true; + rerender(); + expect( + screen.getByRole('switch', { + name: 'Toggle custom automation launch criteria', + }), + ).toBeDisabled(); + }); +}); diff --git a/apps/web/src/components/settings/AutomationLaunchCriteriaExperimentalSetting.tsx b/apps/web/src/components/settings/AutomationLaunchCriteriaExperimentalSetting.tsx new file mode 100644 index 0000000000..776f10e836 --- /dev/null +++ b/apps/web/src/components/settings/AutomationLaunchCriteriaExperimentalSetting.tsx @@ -0,0 +1,28 @@ +'use client'; + +import { Switch, Zap } from '@/components/system'; +import { useAutomationLaunchCriteriaExperiment } from '@/hooks/useAutomationLaunchCriteriaExperiment'; + +import { Section } from './Section'; + +export function AutomationLaunchCriteriaExperimentalSetting() { + const { enabled, isLoading, isUpdating, setEnabled } = + useAutomationLaunchCriteriaExperiment(); + + return ( +
+
+ +

+ Allow custom automations to use plain-language and typed checks before + a run starts. +

+
+
+ ); +} diff --git a/apps/web/src/components/settings/automations/AutomationsSettings.render.client.test.tsx b/apps/web/src/components/settings/automations/AutomationsSettings.render.client.test.tsx index f4c21ce2ad..2190b48fb3 100644 --- a/apps/web/src/components/settings/automations/AutomationsSettings.render.client.test.tsx +++ b/apps/web/src/components/settings/automations/AutomationsSettings.render.client.test.tsx @@ -20,6 +20,7 @@ const state = vi.hoisted(() => ({ ) => void; } | null, isAdmin: true, + automationLaunchCriteriaEnabled: false, catalogQueryOptions: [] as Array<{ enabled?: boolean }>, queriedKeys: [] as unknown[], customAutomationsPending: false, @@ -377,6 +378,7 @@ vi.mock('@tanstack/react-query', () => ({ return { isPending: state.settingsQuery.isPending, data: { + launchCriteriaEnabled: state.automationLaunchCriteriaEnabled, capabilities: state.settingsQuery.data.capabilities, managerSlackChannelId, managerDiscordChannelId, @@ -684,6 +686,130 @@ it('opens the standalone custom editor without querying admin settings', () => { expect(state.queriedKeys).not.toContainEqual(['comms', 'status']); }); +it('saves optional launch criteria when creating a custom automation', () => { + state.automationLaunchCriteriaEnabled = true; + render(); + fireEvent.click(screen.getByRole('button', { name: 'New' })); + fireEvent.change(screen.getByRole('textbox', { name: 'Name' }), { + target: { value: 'Regression scan' }, + }); + fireEvent.change(screen.getByRole('textbox', { name: 'Prompt' }), { + target: { value: 'Check current production issues.' }, + }); + const launchCriteria = screen.getByRole('textbox', { + name: 'Launch criteria (optional)', + }); + expect(launchCriteria).toHaveAttribute('maxLength', '4000'); + fireEvent.change(launchCriteria, { + target: { value: 'Only investigate new production regressions.' }, + }); + fireEvent.click( + screen.getByRole('combobox', { name: 'Preferred environment' }), + ); + fireEvent.click(screen.getByRole('option', { name: 'Let Roomote decide' })); + fireEvent.click(screen.getByRole('button', { name: 'Create' })); + + expect(mutations.updateSettings).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'Regression scan', + launchCriteria: 'Only investigate new production regressions.', + }), + ); +}); + +it('prefills and updates launch criteria when editing a custom automation', async () => { + state.automationLaunchCriteriaEnabled = true; + setRunnableCustomAutomation('Only investigate new checkout regressions.'); + render(); + fireEvent.click( + await screen.findByRole('button', { name: 'Configure Daily scan' }), + ); + const launchCriteria = screen.getByRole('textbox', { + name: 'Launch criteria (optional)', + }); + expect(launchCriteria).toHaveValue( + 'Only investigate new checkout regressions.', + ); + fireEvent.change(launchCriteria, { + target: { value: 'Only investigate regressions affecting active users.' }, + }); + fireEvent.click(screen.getByRole('button', { name: 'Save' })); + + expect(mutations.updateSettings).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'automation-1', + launchCriteria: 'Only investigate regressions affecting active users.', + }), + ); +}); + +it('hides and omits custom automation launch criteria while the experiment is off', () => { + state.automationLaunchCriteriaEnabled = false; + render(); + fireEvent.click(screen.getByRole('button', { name: 'New' })); + + expect( + screen.queryByRole('textbox', { name: 'Launch criteria (optional)' }), + ).not.toBeInTheDocument(); + fireEvent.change(screen.getByRole('textbox', { name: 'Name' }), { + target: { value: 'Unconditional report' }, + }); + fireEvent.change(screen.getByRole('textbox', { name: 'Prompt' }), { + target: { value: 'Summarize current production issues.' }, + }); + fireEvent.click( + screen.getByRole('combobox', { name: 'Preferred environment' }), + ); + fireEvent.click(screen.getByRole('option', { name: 'Let Roomote decide' })); + mutations.updateSettings.mockClear(); + fireEvent.click(screen.getByRole('button', { name: 'Create' })); + + expect(mutations.updateSettings).toHaveBeenCalledOnce(); + expect(mutations.updateSettings.mock.calls[0]?.[0]).not.toHaveProperty( + 'launchCriteria', + ); +}); + +it('lets users toggle an automation with saved criteria while the experiment is off', async () => { + state.automationLaunchCriteriaEnabled = false; + setRunnableCustomAutomation('Only investigate new checkout regressions.'); + render(); + + const toggle = await screen.findByRole('switch', { + name: 'Toggle Daily scan', + }); + mutations.updateSettings.mockClear(); + fireEvent.click(toggle); + + expect(mutations.updateSettings).toHaveBeenCalledOnce(); + expect(mutations.updateSettings.mock.calls[0]?.[0]).toMatchObject({ + id: 'automation-1', + enabled: false, + }); + expect(mutations.updateSettings.mock.calls[0]?.[0]).not.toHaveProperty( + 'launchCriteria', + ); +}); + +it('keeps channel auto-start launch criteria editable with the custom experiment off', async () => { + state.automationLaunchCriteriaEnabled = false; + render(); + fireEvent.click( + await screen.findByRole('button', { + name: /(?:Set up|Configure) Auto-respond to channels/, + }), + ); + + const criteria = await screen.findByRole('textbox', { + name: 'Launch criteria (optional)', + }); + fireEvent.change(criteria, { + target: { value: 'Only start for new incidents.' }, + }); + + expect(criteria).toHaveValue('Only start for new incidents.'); +}); + it('validates required custom automation fields before creating', () => { render(); fireEvent.click(screen.getByRole('button', { name: 'New' })); @@ -794,12 +920,13 @@ function closeAutomationDialog() { fireEvent.click(screen.getByRole('button', { name: 'Close' })); } -function setRunnableCustomAutomation() { +function setRunnableCustomAutomation(launchCriteria?: string) { state.customAutomations = [ { id: 'automation-1', name: 'Daily scan', prompt: 'Find flaky tests.', + ...(launchCriteria !== undefined ? { launchCriteria } : {}), enabled: true, scheduleMode: 'daily', cronExpression: null, @@ -825,6 +952,7 @@ describe('AutomationsSettings', () => { vi.clearAllMocks(); state.nextUpdateSettingsResult = null; state.customAutomationRunPendingId = null; + state.automationLaunchCriteriaEnabled = false; mutations.latestSettingsOptions = null; mutations.latestTriggerOptions = null; mutations.latestCustomTriggerOptions = null; diff --git a/apps/web/src/components/settings/automations/CustomAutomationsSection.tsx b/apps/web/src/components/settings/automations/CustomAutomationsSection.tsx index ed21dc1501..c37f03d5f1 100644 --- a/apps/web/src/components/settings/automations/CustomAutomationsSection.tsx +++ b/apps/web/src/components/settings/automations/CustomAutomationsSection.tsx @@ -18,6 +18,7 @@ import { NO_REPOSITORIES, AUTOMATION_RESULT_PRIORITY_LABELS, AUTOMATION_RESULT_PRIORITIES, + CUSTOM_AUTOMATION_LAUNCH_CRITERIA_MAX_LENGTH, CUSTOM_AUTOMATION_PROMPT_MAX_LENGTH, type AutomationResultPriority, type CustomAutomationScheduleMode, @@ -81,6 +82,7 @@ type ConnectedDestinationProvider = Exclude< type CustomAutomationFormState = { name: string; prompt: string; + launchCriteria: string; enabled: boolean; resultPriority: AutomationResultPriority; scheduleMode: CustomAutomationScheduleMode; @@ -101,6 +103,7 @@ type CustomAutomationFieldErrors = Partial< const EMPTY_FORM: CustomAutomationFormState = { name: '', prompt: '', + launchCriteria: '', enabled: true, resultPriority: 'normal', scheduleMode: 'daily', @@ -312,6 +315,7 @@ function formFromRow( return { name: row.name, prompt: row.prompt, + launchCriteria: row.launchCriteria ?? '', enabled: row.enabled, resultPriority: row.resultPriority ?? 'normal', scheduleMode: row.scheduleMode, @@ -325,12 +329,18 @@ function formFromRow( }; } -function writeInputFromRow(row: CustomAutomationListItem) { +function writeInputFromRow( + row: CustomAutomationListItem, + includeLaunchCriteria: boolean, +) { const target = targetFromRow(row); return { name: row.name, prompt: row.prompt, + ...(includeLaunchCriteria + ? { launchCriteria: row.launchCriteria ?? '' } + : {}), enabled: row.enabled, resultPriority: row.resultPriority ?? 'normal', scheduleMode: row.scheduleMode, @@ -453,6 +463,8 @@ export function CustomAutomationsSection({ const optionsQuery = useQuery( trpc.automations.getCustomAutomationOptions.queryOptions(), ); + const automationLaunchCriteriaEnabled = + optionsQuery.data?.launchCriteriaEnabled === true; const taskModelsQuery = useLaunchTaskModels(); const [editingId, setEditingId] = useState(null); @@ -902,6 +914,9 @@ export function CustomAutomationsSection({ const payload = { name: form.name, prompt: form.prompt, + ...(automationLaunchCriteriaEnabled + ? { launchCriteria: form.launchCriteria.trim() || null } + : {}), enabled: form.enabled, resultPriority: form.resultPriority, scheduleMode: form.scheduleMode, @@ -1008,6 +1023,32 @@ export function CustomAutomationsSection({ ) : null} + {automationLaunchCriteriaEnabled ? ( +
+ +