From 7ad623a4876582aaa1cb80b92f87156c90105497 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Sun, 11 Oct 2026 03:53:09 +0300 Subject: [PATCH] Show campaign A/B results and weighted variations in human output campaign analytics renders the A/B criterion, outcome, holdout and reserve counts and a per-variant table with the winner marked. campaign view (and create) show the variant count and outcome, and the number of weighted message alternatives. campaign create help and the README describe the variants, variantStrategy and messageVariations beta input fields. The published SDK pin types these fields as unknown or omits them, so the CLI narrows them at runtime with local types that accept the SDK main types from polymorfa/sdks#344. --- README.md | 62 ++++ contracts/cli-reference.json | 2 +- src/commands/campaign/create.ts | 10 +- src/services/campaign-experiments.ts | 257 ++++++++++++++++ src/services/campaigns.ts | 6 + test/commands/campaign-experiments.test.ts | 327 +++++++++++++++++++++ 6 files changed, 661 insertions(+), 3 deletions(-) create mode 100644 src/services/campaign-experiments.ts create mode 100644 test/commands/campaign-experiments.test.ts diff --git a/README.md b/README.md index 55f947f..b2d9cbd 100644 --- a/README.md +++ b/README.md @@ -1833,6 +1833,68 @@ text. `campaign analytics` adds follow-up outcomes after the first-message counts, and `campaign recipients list` adds a follow-up status column when a recipient has one. `--json` returns the API response unchanged. +### A/B tests and weighted message variation (beta) + +Both betas require team enrollment and live feature access. A team without +access gets `403 feature_unavailable`. For Messaging campaign creation, use +`campaign create --project-slug` with these fields in the `--input` object. + +- **A/B test:** `variants` holds two to four messages, each with a `key`, a + `label`, an integer `weight` (the weights add up to 100) and a `blueprint`. + `variantStrategy` sets `winnerCriterion` (`delivery`, `read` or `reply`), + `holdoutPercent` (0 to 30), an optional `testSlicePercent` (5 to 50, + default 20), `autoPromote: true` and `testWindowMinutes` (15 to 1,440). An + A/B test cannot use `templateId`. +- **Weighted message variation:** `messageVariations` holds two to five + alternatives, each with a `key`, an integer `weight` from 1 to 99 (the + weights add up to 100) and a `blueprint`. Each recipient receives one + alternative and keeps it across retries. Every alternative must use the same + `{{variable}}` placeholders. Alternatives send only from linked-device + numbers and cannot be combined with `templateId`, composed messages or an + A/B test. + +```json +{ + "name": "October offer", + "variants": [ + { + "key": "a", + "label": "Short", + "weight": 50, + "blueprint": {"version": 2, "source": "Hi {{first_name | there}}!"} + }, + { + "key": "b", + "label": "Long", + "weight": 50, + "blueprint": { + "version": 2, + "source": "Hello {{first_name | there}}, your offer is ready." + } + } + ], + "variantStrategy": { + "winnerCriterion": "read", + "holdoutPercent": 5, + "autoPromote": true, + "testWindowMinutes": 120 + } +} +``` + +At launch, recipients are split into an optional holdout, a test group that +receives the variants and a reserve that waits for the winner. When the test +window ends, the winner is sent to the reserve. If no variant is clearly ahead, +the outcome is inconclusive and the campaign pauses with the reserve unsent. + +`campaign view` shows the variant count and outcome of an A/B test, and the +number of weighted alternatives. `campaign analytics` adds the criterion, the +outcome (the winner, inconclusive with its reason, or no winner yet), the +holdout and reserve counts, and a row per variant with the criterion rate, +sent, assigned, delivered, read and replied counts. The winner row is marked +`(winner)`. `--json` returns the API's `variants`, `variantStrategy`, +`experimentOutcome`, `messageVariations` and `experiment` fields unchanged. + ### Campaign recipients Recipients are appended to a campaign that has not started sending, and read diff --git a/contracts/cli-reference.json b/contracts/cli-reference.json index faa90fd..edb14df 100644 --- a/contracts/cli-reference.json +++ b/contracts/cli-reference.json @@ -16701,7 +16701,7 @@ { "id": "campaign create", "summary": "Create a campaign from one JSON object. Optional beta fields include followUp: one receipt-timed follow-up ({delaySeconds, expirySeconds, message: {type: \"text\", text} or {type: \"template\", templateId}}); teams that are not enrolled receive 403 feature_unavailable.", - "description": "Create a campaign from one JSON object. Optional beta fields include followUp: one receipt-timed follow-up ({delaySeconds, expirySeconds, message: {type: \"text\", text} or {type: \"template\", templateId}}); teams that are not enrolled receive 403 feature_unavailable.", + "description": "Create a campaign from one JSON object. Optional beta fields include followUp: one receipt-timed follow-up ({delaySeconds, expirySeconds, message: {type: \"text\", text} or {type: \"template\", templateId}}); teams that are not enrolled receive 403 feature_unavailable.\n\nFor Messaging campaigns (--project-slug), A/B tests and weighted message variation are betas that require team enrollment and live feature access; a team without access gets 403 feature_unavailable:\n\n- A/B test: \"variants\" (two to four, each with key, label, integer weight and blueprint; weights add up to 100) and \"variantStrategy\" (winnerCriterion delivery, read or reply; holdoutPercent; optional testSlicePercent; autoPromote true; testWindowMinutes). View the results with polymorfa campaign analytics.\n- Weighted message variation: \"messageVariations\", two to five alternatives (key, integer weight, blueprint; weights add up to 100). Each recipient receives one alternative. Linked-device numbers only.\n\nAn A/B test cannot use templateId. messageVariations cannot be combined with templateId, composed messages or an A/B test.", "usage": ["polymorfa campaign create [flags]"], "aliases": [], "hidden": false, diff --git a/src/commands/campaign/create.ts b/src/commands/campaign/create.ts index 8c34947..c2f5085 100644 --- a/src/commands/campaign/create.ts +++ b/src/commands/campaign/create.ts @@ -16,8 +16,14 @@ import {idempotencyKey} from '../../input/idempotency.js' import {createCampaign, renderCampaign} from '../../services/campaigns.js' export default class CampaignCreate extends BaseCommand { - static override description = - 'Create a campaign from one JSON object. Optional beta fields include followUp: one receipt-timed follow-up ({delaySeconds, expirySeconds, message: {type: "text", text} or {type: "template", templateId}}); teams that are not enrolled receive 403 feature_unavailable.' + static override description = `Create a campaign from one JSON object. Optional beta fields include followUp: one receipt-timed follow-up ({delaySeconds, expirySeconds, message: {type: "text", text} or {type: "template", templateId}}); teams that are not enrolled receive 403 feature_unavailable. + +For Messaging campaigns (--project-slug), A/B tests and weighted message variation are betas that require team enrollment and live feature access; a team without access gets 403 feature_unavailable: + +- A/B test: "variants" (two to four, each with key, label, integer weight and blueprint; weights add up to 100) and "variantStrategy" (winnerCriterion delivery, read or reply; holdoutPercent; optional testSlicePercent; autoPromote true; testWindowMinutes). View the results with polymorfa campaign analytics. +- Weighted message variation: "messageVariations", two to five alternatives (key, integer weight, blueprint; weights add up to 100). Each recipient receives one alternative. Linked-device numbers only. + +An A/B test cannot use templateId. messageVariations cannot be combined with templateId, composed messages or an A/B test.` static override flags = { ...dripFlags, diff --git a/src/services/campaign-experiments.ts b/src/services/campaign-experiments.ts new file mode 100644 index 0000000..5dc19ae --- /dev/null +++ b/src/services/campaign-experiments.ts @@ -0,0 +1,257 @@ +import {terminalText} from '../developer/render.js' + +/* + * Campaign A/B tests and weighted message variation (beta). + * + * The published SDK pin types these fields as unknown or omits them, so the + * CLI reads them as unknown values and narrows them here. The shapes follow + * the SDK's CampaignExperimentResults, CampaignExperimentOutcome, + * CampaignVariant and CampaignMessageVariation types. A value that does not + * match is reported as unreadable instead of being printed partially. + */ + +export type ExperimentOutcome = + | {readonly state: 'promoted'; readonly winnerKey: string} + | {readonly reason: string; readonly state: 'inconclusive'} + +export interface ExperimentVariantResult { + readonly assigned: number + readonly delivered: number + readonly key: string + readonly label: string + readonly outcomeRate: number + readonly read: number + readonly replied: number + readonly sent: number + readonly weight: number +} + +export interface ExperimentResults { + readonly criterion: string + readonly holdoutCount: number + readonly outcome: ExperimentOutcome | null + readonly reserveCount: number + readonly variants: readonly ExperimentVariantResult[] +} + +interface VariantSummary { + readonly key: string + readonly label: string +} + +const UNREADABLE = + 'Unreadable in this CLI version; use --json for the full response.' + +const CRITERION_RATE: Readonly> = { + delivery: 'Delivery rate', + read: 'Read rate', + reply: 'Reply rate', +} + +/** Human lines for `campaign analytics`; empty when the campaign is not an A/B test. */ +export function renderExperimentAnalytics(analytics: object): string[] { + const value = field(analytics, 'experiment') + if (value === undefined || value === null) return [] + const experiment = experimentResults(value) + if (experiment === undefined) return ['', `A/B test\t${UNREADABLE}`] + + const rateLabel = + CRITERION_RATE[experiment.criterion] ?? + `${terminalText(experiment.criterion)} rate` + const winnerKey = + experiment.outcome?.state === 'promoted' + ? experiment.outcome.winnerKey + : undefined + + return [ + '', + `A/B criterion\t${rateLabel}`, + `A/B outcome\t${describeOutcome(experiment.outcome, experiment.variants)}`, + `Holdout\t${experiment.holdoutCount}`, + `Reserve\t${experiment.reserveCount}`, + '', + [ + 'Variant', + rateLabel, + 'Sent', + 'Assigned', + 'Delivered', + 'Read', + 'Replied', + ].join('\t'), + ...experiment.variants.map((variant) => + [ + `${variantName(variant)}${variant.key === winnerKey ? ' (winner)' : ''}`, + percent(variant.outcomeRate), + variant.sent, + variant.assigned, + variant.delivered, + variant.read, + variant.replied, + ].join('\t'), + ), + ] +} + +/** Human lines for `campaign view` covering A/B variants and weighted alternatives. */ +export function renderCampaignBetas(campaign: object): string[] { + const lines: string[] = [] + + const variants = field(campaign, 'variants') + if (variants !== undefined && variants !== null) { + const summaries = variantSummaries(variants) + const outcome = field(campaign, 'experimentOutcome') + const parsed = + outcome === undefined || outcome === null + ? null + : experimentOutcome(outcome) + if (summaries === undefined || parsed === undefined) { + lines.push(`A/B test\t${UNREADABLE}`) + } else { + lines.push( + `A/B test\t${summaries.length} ${summaries.length === 1 ? 'variant' : 'variants'}`, + `A/B outcome\t${describeOutcome(parsed, summaries)}`, + ) + } + } + + const variations = field(campaign, 'messageVariations') + if (variations !== undefined && variations !== null) { + lines.push( + Array.isArray(variations) + ? `Message variations\t${variations.length} weighted ${variations.length === 1 ? 'alternative' : 'alternatives'}` + : `Message variations\t${UNREADABLE}`, + ) + } + + return lines +} + +function describeOutcome( + outcome: ExperimentOutcome | null, + variants: readonly VariantSummary[], +): string { + if (outcome === null) return 'No winner yet' + if (outcome.state === 'promoted') { + const winner = variants.find((variant) => variant.key === outcome.winnerKey) + return `Winner: ${winner === undefined ? terminalText(outcome.winnerKey) : variantName(winner)}, promoted to the reserve` + } + return `Inconclusive (${terminalText(outcome.reason)}); the reserve was not sent` +} + +function variantName(variant: VariantSummary): string { + return `${terminalText(variant.label)} [${terminalText(variant.key)}]` +} + +function percent(rate: number): string { + return `${(Math.round(rate * 1000) / 10).toFixed(1)}%` +} + +function experimentResults(value: unknown): ExperimentResults | undefined { + if (!isRecord(value)) return undefined + const {criterion, holdoutCount, reserveCount, variants} = value + const outcome = + value.outcome === null || value.outcome === undefined + ? null + : experimentOutcome(value.outcome) + if ( + typeof criterion !== 'string' || + !isCount(holdoutCount) || + !isCount(reserveCount) || + outcome === undefined || + !Array.isArray(variants) + ) { + return undefined + } + const results: ExperimentVariantResult[] = [] + for (const variant of variants) { + const parsed = variantResult(variant) + if (parsed === undefined) return undefined + results.push(parsed) + } + return { + criterion, + holdoutCount, + outcome, + reserveCount, + variants: results, + } +} + +function variantResult(value: unknown): ExperimentVariantResult | undefined { + if (!isRecord(value)) return undefined + const { + assigned, + delivered, + key, + label, + outcomeRate, + read, + replied, + sent, + weight, + } = value + if ( + typeof key !== 'string' || + typeof label !== 'string' || + !isFiniteNumber(outcomeRate) || + !isFiniteNumber(weight) || + ![assigned, delivered, read, replied, sent].every(isCount) + ) { + return undefined + } + return { + assigned: assigned as number, + delivered: delivered as number, + key, + label, + outcomeRate, + read: read as number, + replied: replied as number, + sent: sent as number, + weight, + } +} + +function experimentOutcome(value: unknown): ExperimentOutcome | undefined { + if (!isRecord(value)) return undefined + if (value.state === 'promoted' && typeof value.winnerKey === 'string') { + return {state: 'promoted', winnerKey: value.winnerKey} + } + if (value.state === 'inconclusive' && typeof value.reason === 'string') { + return {reason: value.reason, state: 'inconclusive'} + } + return undefined +} + +function variantSummaries(value: unknown): VariantSummary[] | undefined { + if (!Array.isArray(value)) return undefined + const summaries: VariantSummary[] = [] + for (const variant of value) { + if ( + !isRecord(variant) || + typeof variant.key !== 'string' || + typeof variant.label !== 'string' + ) { + return undefined + } + summaries.push({key: variant.key, label: variant.label}) + } + return summaries +} + +function field(value: object, name: string): unknown { + return (value as Readonly>)[name] +} + +function isCount(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/services/campaigns.ts b/src/services/campaigns.ts index 58ec7ba..6b27f90 100644 --- a/src/services/campaigns.ts +++ b/src/services/campaigns.ts @@ -1,3 +1,7 @@ +import { + renderCampaignBetas, + renderExperimentAnalytics, +} from './campaign-experiments.js' import { renderDripConfiguration, renderDripStats, @@ -242,6 +246,7 @@ export function renderCampaign(campaign: Campaign): string { `Scheduled\t${formatCampaignTime(campaign.scheduledAt)}`, ...renderFollowUpConfiguration(campaign), ...renderDripConfiguration(campaign), + ...renderCampaignBetas(campaign), ].join('\n') } @@ -259,6 +264,7 @@ export function renderCampaignAnalytics(analytics: CampaignAnalytics): string { ...renderFollowUpAnalytics(analytics), ...renderDripStats(analytics), ...renderFrequencyCapAnalytics(analytics), + ...renderExperimentAnalytics(analytics), ].join('\n') } diff --git a/test/commands/campaign-experiments.test.ts b/test/commands/campaign-experiments.test.ts new file mode 100644 index 0000000..ebb8526 --- /dev/null +++ b/test/commands/campaign-experiments.test.ts @@ -0,0 +1,327 @@ +import {runCommand} from '@oclif/test' +import {afterEach, describe, expect, test} from 'vitest' + +import { + startHttpServer, + withEnvironment, + type TestHttpServer, +} from '../helpers/http-server.js' + +const API_KEY = + 'pmfa_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + +describe('campaign A/B test and weighted variation output (beta)', () => { + let server: TestHttpServer | undefined + + afterEach(async () => { + await server?.close() + server = undefined + }) + + test('renders a promoted A/B test with the winner marked', async () => { + const data = analytics({ + criterion: 'read', + holdoutCount: 50, + outcome: {state: 'promoted', winnerKey: 'b'}, + reserveCount: 700, + variants: [variant('a', 'Short', 0.4), variant('b', 'Long', 0.5625)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toBe( + [ + ...BASE_ANALYTICS_LINES, + '', + 'A/B criterion\tRead rate', + 'A/B outcome\tWinner: Long [b], promoted to the reserve', + 'Holdout\t50', + 'Reserve\t700', + '', + 'Variant\tRead rate\tSent\tAssigned\tDelivered\tRead\tReplied', + 'Short [a]\t40.0%\t80\t100\t75\t32\t6', + 'Long [b] (winner)\t56.3%\t80\t100\t75\t32\t6', + ].join('\n'), + ) + }) + + test('renders an inconclusive A/B test with its reason', async () => { + const data = analytics({ + criterion: 'reply', + holdoutCount: 0, + outcome: {reason: 'insufficient_evidence', state: 'inconclusive'}, + reserveCount: 800, + variants: [variant('a', 'Short', 0.05), variant('b', 'Long', 0.06)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toContain('A/B criterion\tReply rate') + expect(output).toContain( + 'A/B outcome\tInconclusive (insufficient_evidence); the reserve was not sent', + ) + expect(output).toContain( + 'Variant\tReply rate\tSent\tAssigned\tDelivered\tRead\tReplied', + ) + expect(output).not.toContain('(winner)') + }) + + test('renders a running A/B test with no winner yet', async () => { + const data = analytics({ + criterion: 'delivery', + holdoutCount: 10, + outcome: null, + reserveCount: 500, + variants: [variant('a', 'Short', 0.9), variant('b', 'Long', 0.85)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toContain('A/B outcome\tNo winner yet') + expect(output).toContain('Short [a]\t90.0%') + expect(output).not.toContain('(winner)') + }) + + test.each([ + ['null', {experiment: null}], + ['absent', {}], + ])( + 'keeps analytics unchanged when experiment is %s', + async (_name, extra) => { + const output = await humanOutput('campaign analytics campaign_1', { + ...baseAnalytics(), + ...extra, + }) + + expect(output).toBe(BASE_ANALYTICS_LINES.join('\n')) + }, + ) + + test('reports an unreadable experiment instead of printing part of it', async () => { + const output = await humanOutput('campaign analytics campaign_1', { + ...baseAnalytics(), + experiment: {criterion: 'read', variants: 'unexpected'}, + }) + + expect(output).toContain( + 'A/B test\tUnreadable in this CLI version; use --json for the full response.', + ) + }) + + test('escapes control characters in variant labels', async () => { + const output = await humanOutput( + 'campaign analytics campaign_1', + analytics({ + criterion: 'read', + holdoutCount: 0, + outcome: null, + reserveCount: 0, + variants: [variant('a', 'Bad\u001b[31m', 0.1)], + }), + ) + + expect(output).toContain('Bad\\u001b[31m [a]') + expect(output).not.toContain('\u001b') + }) + + test('passes experiment results through unchanged in JSON output', async () => { + const data = analytics({ + criterion: 'read', + holdoutCount: 0, + outcome: {state: 'promoted', winnerKey: 'a'}, + reserveCount: 10, + variants: [variant('a', 'Short', 0.4), variant('b', 'Long', 0.2)], + }) + server = await startHttpServer({body: {data, success: true}}) + + const result = await withApiKey(() => + runCommand('campaign analytics campaign_1 --project-slug launch --json'), + ) + + expect(result.error).toBeUndefined() + expect((JSON.parse(result.stdout) as {data: unknown}).data).toEqual(data) + }) + + test('shows the variant count and outcome on an A/B campaign', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: {state: 'promoted', winnerKey: 'a'}, + variants: [campaignVariant('a', 'Short'), campaignVariant('b', 'Long')], + variantStrategy: { + autoPromote: true, + holdoutPercent: 5, + testWindowMinutes: 120, + winnerCriterion: 'read', + }, + }) + + expect(output).toBe( + [ + ...BASE_CAMPAIGN_LINES, + 'A/B test\t2 variants', + 'A/B outcome\tWinner: Short [a], promoted to the reserve', + ].join('\n'), + ) + }) + + test('shows no winner yet on an undecided A/B campaign', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: null, + variants: [ + campaignVariant('a', 'Short'), + campaignVariant('b', 'Long'), + campaignVariant('c', 'Emoji'), + ], + }) + + expect(output).toContain('A/B test\t3 variants') + expect(output).toContain('A/B outcome\tNo winner yet') + }) + + test('shows the number of weighted alternatives', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + messageVariations: [ + {blueprint: {source: 'Hello!', version: 2}, key: 'hello', weight: 60}, + {blueprint: {source: 'Hi!', version: 2}, key: 'hi', weight: 40}, + ], + variants: null, + }) + + expect(output).toBe( + [ + ...BASE_CAMPAIGN_LINES, + 'Message variations\t2 weighted alternatives', + ].join('\n'), + ) + }) + + test('keeps a broadcast campaign view unchanged', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: null, + messageVariations: null, + variantStrategy: null, + variants: null, + }) + + expect(output).toBe(BASE_CAMPAIGN_LINES.join('\n')) + }) + + test('mentions both beta input fields in create help', async () => { + const result = await runCommand('campaign create --help') + + expect(result.stdout).toContain('variantStrategy') + expect(result.stdout).toContain('messageVariations') + expect(result.stdout).toContain('403 feature_unavailable') + }) + + async function humanOutput(command: string, data: unknown): Promise { + server = await startHttpServer({body: {data, success: true}}) + const result = await withApiKey(() => + runCommand(`${command} --project-slug launch`), + ) + expect(result.error).toBeUndefined() + return result.stdout.trimEnd() + } + + async function withApiKey(run: () => Promise): Promise { + if (server === undefined) throw new Error('Test server was not started.') + return withEnvironment( + { + POLYMORFA_API_KEY: API_KEY, + POLYMORFA_BASE_URL: server.baseUrl, + POLYMORFA_CLIENT_TOKEN: undefined, + }, + run, + ) + } +}) + +const BASE_ANALYTICS_LINES = [ + 'Campaign\tcampaign_1', + 'Recipients\t1000', + 'Sent\t160', + 'Delivered\t150', + 'Read\t64', + 'Failed\t0', + 'Skipped\t50', + 'Responded\t12', + 'Response rate\t0.075', +] + +const BASE_CAMPAIGN_LINES = [ + 'ID\tcampaign_1', + 'Name\tOctober offer', + 'Status\trunning', + 'Recipients\t1000', + 'Sent\t160', + 'Delivered\t150', + 'Read\t64', + 'Failed\t0', + 'Skipped\t50', + 'Scheduled\t2024-08-07T03:07:30.000Z', +] + +function baseAnalytics() { + return { + campaignId: 'campaign_1', + deliveredCount: 150, + failedCount: 0, + readCount: 64, + recipientCount: 1000, + respondedCount: 12, + responseRate: 0.075, + sentCount: 160, + skippedCount: 50, + } +} + +function analytics(experiment: unknown) { + return {...baseAnalytics(), experiment} +} + +function variant(key: string, label: string, outcomeRate: number) { + return { + assigned: 100, + delivered: 75, + key, + label, + outcomeRate, + read: 32, + replied: 6, + sent: 80, + weight: 50, + } +} + +function campaignVariant(key: string, label: string) { + return { + blueprint: {source: `Message ${key}`, version: 2}, + key, + label, + weight: 50, + } +} + +function campaign() { + return { + completedAt: null, + createdAt: 1_723_000_000_000, + deliveredCount: 150, + failedCount: 0, + id: 'campaign_1', + launchedAt: 1_723_000_100_000, + name: 'October offer', + readCount: 64, + recipientCount: 1000, + recipientListId: 'recipients_1', + scheduledAt: 1_723_000_050_000, + sentCount: 160, + skippedCount: 50, + status: 'running', + templateId: null, + updatedAt: 1_723_000_200_000, + } +}