diff --git a/TOOLS.md b/TOOLS.md index 14645b1..2d5c530 100644 --- a/TOOLS.md +++ b/TOOLS.md @@ -1,4 +1,4 @@ -# Available Iterable MCP Tools (109 tools) +# Available Iterable MCP Tools (118 tools) **Legend:** - 🔒 = Requires enabling user PII access @@ -50,11 +50,20 @@ - **track_bulk_events** 🔒✏️✉️: Track multiple events in a single request for better performance - **track_event** 🔒✏️✉️: Track a custom event for a user -## Experiments (4 tools) +## Experiments (13 tools) +- **cancel_experiment** ✏️: Cancel a running or winner_found campaign experiment without declaring a winner. +- **copy_experiment_variant** ✏️: Copy a template into a new variant on a draft or ready campaign experiment. +- **create_experiment** ✏️: Create a draft campaign experiment. The campaign template becomes the control. +- **declare_experiment_winner** ✏️✉️: Declare a winning variant and end a running or winner_found campaign experiment. +- **delete_experiment** ✏️: Delete a draft, ready, or errored campaign experiment. - **get_experiment**: Get detailed information about a specific experiment by ID, including variants summary and constraints - **get_experiment_metrics**: Get experiment metrics for A/B testing analysis (currently supports email experiments only) +- **get_experiment_totals**: Get lifetime send and conversion totals for a campaign experiment. Holdout is omitted, missing counts are 0, and lift and confidence are not included. +- **get_experiment_trends**: Get the performance time series for a campaign experiment. Provide both startDateTime and endDateTime, or omit both to use the run window. The span is at most 31 days; a longer default window is clipped to the last 31 days. - **get_experiment_variants**: Get variant content for an experiment, including subject lines, preheaders, HTML source, and plain text - **list_experiments**: List experiments with optional filtering by campaign, status, and date range. Supports pagination. +- **start_experiment** ✏️✉️: Start a draft or ready campaign experiment. +- **update_experiment_settings** ✏️: Update settings on a draft or ready campaign experiment. Omit a field to leave it unchanged. ## Journeys (2 tools) - **get_journeys**: Get journeys (workflows) with optional pagination and state filtering diff --git a/package.json b/package.json index a4b42aa..99bb0e9 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ }, "dependencies": { "@alcyone-labs/zod-to-json-schema": "4.0.10", - "@iterable/api": "0.12.0", + "@iterable/api": "0.13.0", "@modelcontextprotocol/sdk": "1.18.1", "@primno/dpapi": "2.0.1", "@types/json-schema": "7.0.15", diff --git a/src/tool-filter.ts b/src/tool-filter.ts index 8376dc3..b706853 100644 --- a/src/tool-filter.ts +++ b/src/tool-filter.ts @@ -13,13 +13,18 @@ export const NON_PII_TOOLS: Set = new Set([ "archive_campaigns", "bulk_delete_catalog_items", "cancel_campaign", + "cancel_experiment", + "copy_experiment_variant", "create_blast_campaign", + "create_experiment", "create_triggered_campaign", "create_catalog", "create_list", "create_snippet", "deactivate_triggered_campaign", + "declare_experiment_winner", "delete_catalog", + "delete_experiment", "delete_catalog_item", "delete_list", "delete_snippet", @@ -36,6 +41,8 @@ export const NON_PII_TOOLS: Set = new Set([ "get_email_template", "get_experiment", "get_experiment_metrics", + "get_experiment_totals", + "get_experiment_trends", "get_experiment_variants", "get_inapp_template", "get_journeys", @@ -57,11 +64,13 @@ export const NON_PII_TOOLS: Set = new Set([ "replace_catalog_item", "schedule_campaign", "send_campaign", + "start_experiment", "trigger_campaign", "update_catalog_field_mappings", "partial_update_catalog_items", "replace_catalog_items", "update_email_template", + "update_experiment_settings", "update_inapp_template", "update_push_template", "update_sms_template", @@ -87,6 +96,8 @@ export const READ_ONLY_TOOLS: Set = new Set([ "get_embedded_messages", "get_experiment", "get_experiment_metrics", + "get_experiment_totals", + "get_experiment_trends", "get_experiment_variants", "get_export_files", "get_export_jobs", @@ -125,6 +136,8 @@ export const SEND_TOOLS: Set = new Set([ "send_campaign", "trigger_campaign", "schedule_campaign", + "start_experiment", + "declare_experiment_winner", // Triggered campaigns can cause sends upon activation; block unless explicitly allowed "activate_triggered_campaign", // Journey triggers enqueue users which may send diff --git a/src/tools/experiments.ts b/src/tools/experiments.ts index b2d1271..d82abc7 100644 --- a/src/tools/experiments.ts +++ b/src/tools/experiments.ts @@ -4,10 +4,17 @@ import type { IterableClient } from "@iterable/api"; import { + CopyExperimentVariantParamsSchema, + CreateExperimentParamsSchema, + DeclareExperimentWinnerParamsSchema, + ExperimentIdParamsSchema, GetExperimentMetricsParamsSchema, GetExperimentParamsSchema, + GetExperimentTotalsParamsSchema, + GetExperimentTrendsParamsSchema, GetExperimentVariantsParamsSchema, ListExperimentsParamsSchema, + UpdateExperimentSettingsParamsSchema, } from "@iterable/api"; import type { Tool } from "@modelcontextprotocol/sdk/types.js"; @@ -43,5 +50,66 @@ export function createExperimentTools(client: IterableClient): Tool[] { schema: GetExperimentMetricsParamsSchema, execute: (params) => client.getExperimentMetrics(params), }), + createTool({ + name: "get_experiment_totals", + description: + "Get lifetime send and conversion totals for a campaign experiment. Holdout is omitted, missing counts are 0, and lift and confidence are not included.", + schema: GetExperimentTotalsParamsSchema, + execute: (params) => client.getExperimentTotals(params), + }), + createTool({ + name: "get_experiment_trends", + description: + "Get the performance time series for a campaign experiment. Provide both startDateTime and endDateTime, or omit both to use the run window. The span is at most 31 days; a longer default window is clipped to the last 31 days.", + schema: GetExperimentTrendsParamsSchema, + execute: (params) => client.getExperimentTrends(params), + }), + createTool({ + name: "create_experiment", + description: + "Create a draft campaign experiment. The campaign template becomes the control.", + schema: CreateExperimentParamsSchema, + execute: (params) => client.createExperiment(params), + }), + createTool({ + name: "copy_experiment_variant", + description: + "Copy a template into a new variant on a draft or ready campaign experiment.", + schema: CopyExperimentVariantParamsSchema, + execute: (params) => client.copyExperimentVariant(params), + }), + createTool({ + name: "update_experiment_settings", + description: + "Update settings on a draft or ready campaign experiment. Omit a field to leave it unchanged.", + schema: UpdateExperimentSettingsParamsSchema, + execute: (params) => client.updateExperimentSettings(params), + }), + createTool({ + name: "start_experiment", + description: "Start a draft or ready campaign experiment.", + schema: ExperimentIdParamsSchema, + execute: (params) => client.startExperiment(params), + }), + createTool({ + name: "cancel_experiment", + description: + "Cancel a running or winner_found campaign experiment without declaring a winner.", + schema: ExperimentIdParamsSchema, + execute: (params) => client.cancelExperiment(params), + }), + createTool({ + name: "declare_experiment_winner", + description: + "Declare a winning variant and end a running or winner_found campaign experiment.", + schema: DeclareExperimentWinnerParamsSchema, + execute: (params) => client.declareExperimentWinner(params), + }), + createTool({ + name: "delete_experiment", + description: "Delete a draft, ready, or errored campaign experiment.", + schema: ExperimentIdParamsSchema, + execute: (params) => client.deleteExperiment(params), + }), ]; } diff --git a/tests/unit/experiments-tools.test.ts b/tests/unit/experiments-tools.test.ts new file mode 100644 index 0000000..cc63972 --- /dev/null +++ b/tests/unit/experiments-tools.test.ts @@ -0,0 +1,296 @@ +import { createIterableError, IterableClient } from "@iterable/api"; +import { beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { McpError } from "@modelcontextprotocol/sdk/types.js"; + +import { createExperimentTools } from "../../src/tools/experiments.js"; + +const asyncMock = () => jest.fn<(...args: unknown[]) => Promise>(); + +const mockAxiosInstance = { + get: asyncMock(), + post: asyncMock(), + put: asyncMock(), + delete: asyncMock(), + patch: asyncMock(), +}; + +type ToolResult = { + content: Array<{ type: string; text: string }>; +}; + +const experiment = { + id: 884102, + status: "draft", + channelType: "email", + experimentType: "SubjectLine", + allocationMode: "even_split", + sizing: { + holdoutPercentage: null, + perVariantPercentage: null, + }, + constraints: null, + meta: { + name: "Welcome Experiment", + conversionMetrics: ["opens"], + campaignId: 129500, + projectId: 1, + orgId: 2, + }, + variants: [ + { + id: 0, + name: "Control", + value: { templateId: 55 }, + currentPercentage: 50, + isWinner: false, + isControl: true, + }, + ], +}; + +const totals = { + id: 884102, + status: "running", + variants: [ + { + id: 0, + name: "Control", + isControl: true, + isWinner: false, + metrics: { sends: 5000, emailOpen: 1250, purchase: 0 }, + }, + ], +}; + +function toolHandler(client: IterableClient, name: string) { + const tool = createExperimentTools(client).find((item) => item.name === name); + if (!tool || !("handler" in tool) || typeof tool.handler !== "function") { + throw new Error(`missing tool ${name}`); + } + return tool.handler as (args: unknown) => Promise; +} + +function restError(status: number, error: string) { + return createIterableError({ + response: { status, data: { error } }, + config: { url: "/api/experiments/884102" }, + }); +} + +async function errorBody(client: IterableClient, name: string, args: unknown) { + const result = await toolHandler(client, name)(args); + return JSON.parse(result.content[0]?.text ?? "{}") as { + statusCode: number; + rawResponse: { error: string }; + }; +} + +describe("experiment tools", () => { + let client: IterableClient; + + beforeEach(() => { + jest.clearAllMocks(); + client = new IterableClient( + { + apiKey: "test-key", + baseUrl: "https://api.iterable.com", + timeout: 30000, + }, + mockAxiosInstance as never + ); + }); + + it("returns lifetime totals", async () => { + mockAxiosInstance.get.mockResolvedValue({ data: totals }); + + const result = await toolHandler( + client, + "get_experiment_totals" + )({ experimentId: 884102 }); + + expect(mockAxiosInstance.get).toHaveBeenCalledWith( + "/api/experiments/884102/totals" + ); + expect(JSON.parse(result.content[0]?.text ?? "{}")).toEqual(totals); + }); + + it.each([ + { status: 400, error: "Journey experiments are not supported" }, + { status: 404, error: "Experiment 884102 not found" }, + ])("returns totals status $status for $error", async ({ status, error }) => { + mockAxiosInstance.get.mockRejectedValue(restError(status, error)); + + const body = await errorBody(client, "get_experiment_totals", { + experimentId: 884102, + }); + + expect(body.statusCode).toBe(status); + expect(body.rawResponse).toEqual({ error }); + }); + + it("returns a trends series", async () => { + const trends = { + id: 884102, + status: "running", + interval: "day", + startDateTime: "2024-01-01T00:00:00.000Z", + endDateTime: "2024-01-07T00:00:00.000Z", + variants: [], + }; + mockAxiosInstance.get.mockResolvedValue({ data: trends }); + + const result = await toolHandler( + client, + "get_experiment_trends" + )({ + experimentId: 884102, + startDateTime: "2024-01-01T00:00:00.000Z", + endDateTime: "2024-01-07T00:00:00.000Z", + }); + + expect(JSON.parse(result.content[0]?.text ?? "{}")).toEqual(trends); + }); + + it("rejects a trends request that sets only one datetime", async () => { + await expect( + toolHandler( + client, + "get_experiment_trends" + )({ + experimentId: 884102, + startDateTime: "2024-01-01T00:00:00.000Z", + }) + ).rejects.toBeInstanceOf(McpError); + expect(mockAxiosInstance.get).not.toHaveBeenCalled(); + }); + + it.each([ + { status: 400, error: "Journey experiments are not supported" }, + { status: 400, error: "Date range cannot exceed 31 days" }, + { status: 400, error: "Date range is outside the experiment run window" }, + { status: 404, error: "Experiment 884102 not found" }, + ])("returns trends status $status for $error", async ({ status, error }) => { + mockAxiosInstance.get.mockRejectedValue(restError(status, error)); + + const body = await errorBody(client, "get_experiment_trends", { + experimentId: 884102, + }); + + expect(body.statusCode).toBe(status); + expect(body.rawResponse.error).toBe(error); + }); + + it.each([ + { + name: "create_experiment", + method: "post", + toolArgs: { campaignId: 129500, experimentType: "SubjectLine" }, + url: "/api/experiments", + body: { campaignId: 129500, experimentType: "SubjectLine" }, + }, + { + name: "copy_experiment_variant", + method: "post", + toolArgs: { experimentId: 884102, copyFromTemplateId: 55 }, + url: "/api/experiments/884102/variants", + body: { copyFromTemplateId: 55 }, + }, + { + name: "update_experiment_settings", + method: "patch", + toolArgs: { experimentId: 884102, evenlySplitVariations: true }, + url: "/api/experiments/884102/settings", + body: { evenlySplitVariations: true }, + }, + { + name: "declare_experiment_winner", + method: "post", + toolArgs: { experimentId: 884102, variantId: 2 }, + url: "/api/experiments/884102/winner", + body: { variantId: 2 }, + }, + { + name: "start_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + url: "/api/experiments/884102/start", + body: {}, + }, + { + name: "cancel_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + url: "/api/experiments/884102/cancel", + body: {}, + }, + { + name: "delete_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + url: "/api/experiments/884102/delete", + body: {}, + }, + ] as const)( + "$name sends $url", + async ({ name, method, toolArgs, url, body }) => { + mockAxiosInstance[method].mockResolvedValue({ data: experiment }); + + const result = await toolHandler(client, name)(toolArgs); + + expect(mockAxiosInstance[method]).toHaveBeenCalledWith(url, body); + expect(JSON.parse(result.content[0]?.text ?? "{}").id).toBe(884102); + } + ); + + it.each([ + { + name: "create_experiment", + method: "post", + toolArgs: { campaignId: 129500, experimentType: "SubjectLine" }, + }, + { + name: "copy_experiment_variant", + method: "post", + toolArgs: { experimentId: 884102, copyFromTemplateId: 55 }, + }, + { + name: "update_experiment_settings", + method: "patch", + toolArgs: { experimentId: 884102, holdoutSettings: null }, + }, + { + name: "start_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + }, + { + name: "cancel_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + }, + { + name: "declare_experiment_winner", + method: "post", + toolArgs: { experimentId: 884102, variantId: 2 }, + }, + { + name: "delete_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + }, + ] as const)( + "$name returns 400, 404, and 409", + async ({ name, method, toolArgs }) => { + for (const [status, error] of [ + [400, "Journey experiments are not supported"], + [404, "Experiment 884102 not found"], + [409, "Experiment cannot be changed in its current status"], + ] as const) { + mockAxiosInstance[method].mockRejectedValue(restError(status, error)); + const body = await errorBody(client, name, toolArgs); + expect(body.statusCode).toBe(status); + expect(body.rawResponse.error).toBe(error); + } + } + ); +}); diff --git a/tests/unit/tools.test.ts b/tests/unit/tools.test.ts index f008b92..1b83772 100644 --- a/tests/unit/tools.test.ts +++ b/tests/unit/tools.test.ts @@ -15,19 +15,24 @@ const EXPECTED_TOOLS = [ "bulk_update_users", "cancel_campaign", "cancel_email", + "cancel_experiment", "cancel_export_job", "cancel_in_app", "cancel_push", "cancel_sms", "cancel_web_push", "cancel_whatsapp", + "copy_experiment_variant", "create_blast_campaign", + "create_experiment", "create_triggered_campaign", "create_catalog", "create_list", "create_snippet", "deactivate_triggered_campaign", + "declare_experiment_winner", "delete_catalog", + "delete_experiment", "delete_catalog_item", "delete_list", "delete_snippet", @@ -48,6 +53,8 @@ const EXPECTED_TOOLS = [ "get_embedded_messages", "get_experiment", "get_experiment_metrics", + "get_experiment_totals", + "get_experiment_trends", "get_experiment_variants", "get_export_files", "get_export_jobs", @@ -89,6 +96,7 @@ const EXPECTED_TOOLS = [ "send_sms_template_proof", "send_web_push", "send_whatsapp", + "start_experiment", "start_export_job", "subscribe_to_list", "subscribe_user_by_email", @@ -105,6 +113,7 @@ const EXPECTED_TOOLS = [ "replace_catalog_items", "update_email", "update_email_template", + "update_experiment_settings", "update_inapp_template", "update_push_template", "update_sms_template", @@ -177,7 +186,7 @@ describe("Tool Modules", () => { // Should have a reasonable number of tools (at least 107, allowing for growth) expect(allTools.length).toBeGreaterThanOrEqual(107); - expect(allTools.length).toBeLessThan(115); // Sanity check + expect(allTools.length).toBeLessThan(130); // Sanity check }); it("should have tools from all categories", () => {