diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index ad80c5d..bca1f3f 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -62,7 +62,20 @@ const HOSTED_ACCESS_TOKEN = 'sbp_' + '0'.repeat(40); const rawArgs = process.argv.slice(2); const args = new Set(rawArgs); -const FORCE = !args.has('--skip-existing'); +/** + * Re-run cells that already have results, which is the default. + * + * Deliberately kept that way. A stale row surviving a code change is its own + * hazard, and a live one: `outpost-001`'s scorer, prompt and seed were all + * corrected on 24 August, so keeping its old rows would have published results + * scored by code that no longer exists. + * + * What was missing was a way to *resume*. Three runs were interrupted on 21-22 + * August — a wiped config, exhausted credits, a stopped Docker daemon — and + * each time the obvious command re-ran and re-paid for work already done. Use + * `--resume` for that. `--skip-existing` is kept as its older name. + */ +const FORCE = !(args.has('--skip-existing') || args.has('--resume')); const SMOKE = args.has('--smoke'); const DRY = args.has('--dry'); const EXPERIMENT_FILTERS = readRepeatedFlag(rawArgs, 'experiment').map( @@ -104,6 +117,21 @@ function readFlag(name: string): string | undefined { * skipped for exactly the reason its scorer would otherwise have had to detect * and report. */ +/** + * Did the agent's process die, as opposed to finishing or running out of budget? + * + * Deliberately a deny-list of the two shapes that mean "the process fell over" + * rather than an allow-list of good ones. Runners emit reasons from several + * sources — a CLI subtype, an AI SDK finish reason, or the exit code — and a + * reason nobody has catalogued yet is more likely to be a new legitimate + * outcome than a new kind of crash. Erring the other way would start silently + * discarding valid rows. + */ +function isDeadProcess(reason: string | undefined): boolean { + if (!reason) return false; + return reason === 'error' || reason.startsWith('error_exit_'); +} + function unmetRequirements(ev: EvalManifest): string[] { const available: Record = { outpost: Boolean(process.env.OUTPOST_API_KEY), @@ -463,6 +491,33 @@ async function runOne( ); } + // A process that died is not an answer, however much transcript it left. + // + // The guard above only catches an *empty* one, so a container killed + // mid-flight came through with a populated transcript and a plausible + // partial score. One did on 21 August — `error_exit_255`, fifteen tool + // calls, written as a 2/6 agent failure on `outpost-004` — and was spotted + // only because a baseline failing after three passes looked odd. That is + // the 13 August incident wearing different clothes. + // + // Budget exhaustion is *not* in this category and stays scoreable: + // `timeout`, `max_steps` and `error_max_turns` all mean the agent had its + // run and used it up, which is a result about the agent. + // + // Known caveat: `processStopReason` maps exit 137 to `timeout`, and 137 is + // SIGKILL — which is also what a stopping Docker daemon sends. A container + // killed that way is indistinguishable here from one we timed out + // ourselves, and would still be scored. Telling them apart needs the + // harness's own elapsed time, which this layer does not have. + if (isDeadProcess(run.stoppedReason)) { + throw new Error( + `${expName} x ${ev.id}: the agent process did not exit cleanly ` + + `(stoppedReason: ${run.stoppedReason}), so its transcript is not an ` + + 'answer and was not scored. Re-run the cell. If this reason is in fact ' + + 'a legitimate agent outcome, add it to `isDeadProcess`.' + ); + } + last = await (scorer as ToolScorer)({ ...session.scoringContext, toolCalls: run.toolCalls, @@ -681,7 +736,10 @@ async function main() { for (const ev of suiteFiltered) { const out = resultPath(name, ev); if (!FORCE && existsSync(out)) { - console.log(`SKIP ${name} x ${ev.id} (already ran)`); + console.log( + `SKIP ${name} x ${ev.id} (already ran; --resume). Delete its file in ` + + '.eval-runs to force it, and do that if the scorer has changed since.' + ); continue; } if (config.skipEval?.(ev)) { diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 722bbda..66f7739 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1455,7 +1455,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T15:54:51.689Z", + "ranAt": "2026-08-25T10:00:40.539Z", "stage": "build", "product": [ "outpost" @@ -1498,158 +1498,69 @@ "calls": [ { "source": "web_fetch", - "query": "List any documentation pages related to alerts, notifications, destination disabled, consecutive failures, or automatic destination disabling.", + "query": "List any pages related to alerts, notifications, destination disabled events, or consecutive delivery failure thresholds.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], - "resultChars": 779 + "resultChars": 641 }, { "source": "web_fetch", - "query": "Explain how operator events/alerts work in Outpost, especially: how to configure an alert URL/webhook for when a destination is disabled, and any settings for consecutive delivery failure thresholds or warnings before a destination is disabled. Include exact config keys, env vars, or API fields.", + "query": "Give the full details on: what operator events exist (especially destination disabled / deactivated events, and consecutive delivery failure events), how to configure the endpoint/URL that receives them, what config options exist (env vars or API), and whether there is a threshold for consecutive failures before disabling and how to configure alerts before that threshold is hit.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], - "resultChars": 1881 + "resultChars": 1782 }, { "source": "web_fetch", - "query": "List every page URL related to: API reference, admin API, organization settings, project settings, or alert configuration via API (not just self-hosted env vars).", + "query": "List all pages/URLs related to monitoring, managed deployment settings, alerts configuration, or admin API for configuring operator event sinks.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], - "resultChars": 1247 + "resultChars": 1319 }, { "source": "web_fetch", - "query": "This page mentions 'dashboard settings for managed Outpost'. Explain exactly how managed (Hookdeck Cloud) Outpost customers configure operator alerts / alert URL / consecutive failure thresholds — via dashboard, API, or otherwise. Quote any relevant sections about OPERATION_EVENTS_TOPICS, OPERATION_EVENTS_HTTP_URL, ALERT_CONSECUTIVE_FAILURE_COUNT, ALERT_AUTO_DISABLE_DESTINATION and how these map to managed project settings.", + "query": "Explain what \"dashboard settings for managed Outpost\" means for operator events / alerts. How does a managed Outpost user (via console.hookdeck.com) configure the HTTP URL destination for operator events like alert.destination.disabled and alert.destination.consecutive_failure? Is it done through the Hookdeck Console UI, or via an API call? Give exact steps or API endpoint if documented.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration.md" } ], - "resultChars": 950 + "resultChars": 748 }, { "source": "web_fetch", - "query": "Quote any endpoints related to configuring alerts, operator events, webhook alert URLs, consecutive failure thresholds, or destination auto-disable settings at the project level. Include exact HTTP methods, paths, and request/response body fields.", + "query": "Output the raw markdown content verbatim, especially the section about managed deployments / Hookdeck Monitoring settings and how to configure them in the console.", "hasContent": true, - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 1184 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | head -300", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 6966 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | grep -n -i -A 60 \"## Config\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 3596 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | sed -n '896,960p'", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 3133 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/features/operator-events.md", "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], - "resultChars": 8597 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/platform/outpost-projects.md", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/outpost-projects.md" - } - ], - "resultChars": 5841 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | grep -n -i \"OPERATION\\|EVENTS_TOPICS\\|EVENTS_HTTP\\|SINK\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 31 - }, - { - "source": "shell_fetch", - "query": "curl -s \"https://hookdeck.com/docs/outpost/llms.txt\" | grep -i \"operator\\|monitoring\\|alert\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 171 - }, - { - "source": "shell_fetch", - "query": "curl -s \"https://hookdeck.com/docs/llms.txt\" 2>/dev/null | grep -i \"operator\\|outpost\\|monitoring\" ", - "pages": [ - { - "url": "https://hookdeck.com/docs/llms.txt" - } - ], - "resultChars": 101 + "resultChars": 1067 }, { - "source": "shell_fetch", - "query": "curl -s \"https://hookdeck.com/docs/llms.txt\" 2>/dev/null | head -100", - "pages": [ - { - "url": "https://hookdeck.com/docs/llms.txt" - } - ], - "resultChars": 9141 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | sed -n '/### Destination object/,/### List Destinations/p' | head -100", + "source": "web_fetch", + "query": "Find any mention of a \"Config API\" for configuring operator event topics, sinks, or alert thresholds. Give exact endpoint paths, methods, and example request bodies if present.", + "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" + "url": "https://hookdeck.com/docs/outpost/api" } ], - "resultChars": 73 + "resultChars": 873 } ] }, @@ -1668,7 +1579,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:30:15.999Z", + "ranAt": "2026-08-25T10:02:30.384Z", "stage": "build", "product": [ "outpost" @@ -1712,6 +1623,7 @@ "outpost" ], "loaded": [ + "hookdeck", "event-gateway", "outpost" ] @@ -3676,7 +3588,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T15:50:09.305Z", + "ranAt": "2026-08-25T09:44:21.279Z", "stage": "build", "product": [ "outpost" @@ -3686,23 +3598,12 @@ ], "suite": "benchmark", "gatedBy": "discovery", - "passed": true, + "passed": false, "checks": [ { - "name": "alerts are delivered to the endpoint that was asked for", - "passed": true - }, - { - "name": "the disabled-destination alert is subscribed", - "passed": true - }, - { - "name": "the alerting destination is enabled", - "passed": true - }, - { - "name": "the build-up to a disable is subscribed too", - "passed": true + "name": "alerts are delivered somewhere", + "passed": false, + "notes": "no operator events destination exists, so nothing is emitted when a customer's destination is disabled — the same silence that ended the last incident" } ], "skills": { @@ -3713,348 +3614,715 @@ "calls": [ { "source": "web_fetch", - "query": "Find documentation about alerts, notifications, or webhooks for when a destination is disabled or delivery failures occur. Also find the base API URL and authentication method.", + "query": "I need to find how to configure Outpost to send alerts when a destination is disabled due to consecutive delivery failures, and how to configure the consecutive failure threshold. Look for information about admin/operator webhooks, event topics like destination.disabled, and alert configuration.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/references/api" + "url": "https://hookdeck.com/docs/outpost/references/api/config" } ], "resultChars": 215 }, { "source": "web_fetch", - "query": "Give me an overview of the docs site structure, especially anything about alerts, notifications, consecutive failures, destination disabled events, or admin/operator webhooks.", + "query": "Summarize the documentation structure. I'm looking for information about: 1) alerting when a destination is disabled, 2) consecutive delivery failure thresholds, 3) internal/admin webhooks for Outpost's own lifecycle events.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost" } ], - "resultChars": 1254 + "resultChars": 1020 }, { "source": "web_fetch", - "query": "Give full details on alert configuration: how to set up alerts for destination disabled events and consecutive delivery failures, including API endpoints, request/response schemas, thresholds, and debouncing.", + "query": "Find details about alert configuration: consecutive failure thresholds, auto-disabling of destinations, and how to set up a webhook/callback URL to receive alerts when a destination is disabled or when failures accumulate.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/alerts" + "url": "https://hookdeck.com/docs/outpost/overview/alerts" } ], - "resultChars": 1770 + "resultChars": 215 }, { "source": "web_fetch", - "query": "Give full details on alert configuration: how to set up alerts for destination disabled events and consecutive delivery failures, including API endpoints, request/response schemas, thresholds, and debouncing.", + "query": "Find configuration options related to alerts, alert webhook URL, consecutive failure count/threshold for auto-disabling destinations, and callback/notification URLs.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/guides/alerts" + "url": "https://hookdeck.com/docs/outpost/references/config" } ], "resultChars": 215 }, { - "source": "web_fetch", - "query": "Give full details on operator events / alerts configuration: API endpoints, request/response schemas for configuring the sink, topics list, thresholds, and how this works for managed Hookdeck Outpost projects vs self-hosted.", - "hasContent": true, + "source": "web_search", + "query": "Hookdeck Outpost \"alert\" destination disabled consecutive failures configuration site:hookdeck.com", + "hasContent": false, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/references/operator-events" + "url": "https://hookdeck.com/outpost/guides/webhook-alerting-and-notifications", + "title": "Why Alerting and Notifications Are Necessary for Outbound Webhook Providers" + }, + { + "url": "https://hookdeck.com/webhooks/platforms/hookdeck-outpost-vs-hook0-webhook-sending-comparison", + "title": "Hookdeck Outpost vs Hook0: Webhook Sending Compared" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/evaluating-webhook-infrastructure-for-sending-webhooks", + "title": "Evaluating Webhook Infrastructure for Sending Webhooks" + }, + { + "url": "https://hookdeck.com/docs/hookdeck-basics", + "title": "Hookdeck Basics | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/webhooks/platforms/how-to-solve-woocommerce-5-delivery-failure-webhook-disabling", + "title": "How to Solve WooCommerce's 5 Delivery Failure Webhook Disabling" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-cli-v2", + "title": "Hookdeck CLI v2: AI agents can query, inspect, and debug webhooks via MCP" + }, + { + "url": "https://hookdeck.com/docs/use-cases/receive-webhooks", + "title": "Receive and Process Webhooks with Hookdeck | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/issue-triggers", + "title": "Issue Triggers | Hookdeck Event Gateway Docs" } ], - "resultChars": 215 + "resultChars": 2897 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost admin API destination.disabled webhook topics", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-go-guide", + "title": "Send Webhooks with Go" + }, + { + "url": "https://hookdeck.com/outpost", + "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" + }, + { + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2587 }, { "source": "web_fetch", - "query": "List all API endpoints available, especially anything related to admin, operator events, alerts, settings, or topics configuration. Include base URL and auth header format.", + "query": "Find all configuration variables related to alerts: ALERT_CALLBACK_URL, ALERT_CONSECUTIVE_FAILURE_COUNT, ALERT_AUTO_DISABLE_DESTINATION and any others. Give exact variable names, defaults, and whether they can be set via API vs only environment variables/deployment config.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/references/api-reference" + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration" } ], - "resultChars": 215 + "resultChars": 1292 }, { "source": "web_search", - "query": "Hookdeck Outpost \"operator events\" OR \"alert.destination.disabled\" API documentation", + "query": "Hookdeck Outpost Config API PATCH /config alert callback url consecutive failure count", "hasContent": false, "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/outpost/guides/webhook-alerting-and-notifications", + "title": "Why Alerting and Notifications Are Necessary for Outbound Webhook Providers" + }, + { + "url": "https://hookdeck.com/outpost/guides/why-your-platform-needs-outbound-webhooks-and-how-to-ship-them", + "title": "Why Your Platform Needs Outbound Webhooks" + }, { "url": "https://hookdeck.com/docs/outpost/features/event-delivery", "title": "Event Delivery & Retries | Outpost Docs" }, { - "url": "https://hookdeck.com/docs/platform/outpost-projects", - "title": "Outpost Projects | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/docs/hookdeck-basics", + "title": "Hookdeck Basics | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", - "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" } ], - "resultChars": 2166 + "resultChars": 2416 }, { "source": "web_fetch", - "query": "Give full details on operator events, alerts for destination disabled and consecutive failures, how to configure the sink/topics via API, thresholds, and event payload schemas.", + "query": "Show the full Config API reference - what fields exist, including alert configuration (callback_url, consecutive_failure_count, auto_disable_destination), and whether this is settable via REST API with an API key.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/event-delivery" + "url": "https://hookdeck.com/docs/outpost/references/api/config" + } + ], + "resultChars": 215 + }, + { + "source": "web_search", + "query": "Outpost Hookdeck API base URL api.outpost.hookdeck.com /api/v1 reference", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-v0-4-0-released", + "title": "Outpost v0.4.0 Released: Azure Service Bus Support + Full Feature Overview" } ], - "resultChars": 1334 + "resultChars": 2205 }, { "source": "web_fetch", - "query": "Give full details on how Outpost projects work within Hookdeck, especially Monitoring settings, operator events, alerts configuration, and any API endpoints for configuring alert topics/sinks.", + "query": "Find the Outpost REST API base URL and list of endpoints, especially anything related to configuration, alerts, topics, or admin settings.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/platform/outpost-projects" + "url": "https://hookdeck.com/docs/outpost/references/api" } ], - "resultChars": 1389 + "resultChars": 215 }, { "source": "web_fetch", - "query": "Give complete details on operator events: full list of event topics (especially alert.destination.disabled and alert.destination.consecutive_failure), how to configure the sink URL for managed Hookdeck Outpost projects (via API or dashboard), thresholds, event payload schemas, and any authentication/signing details.", + "query": "Explain the operator events feature: what topics are available (e.g. destination.disabled, alert events), the payload format, and how to configure OPERATOR_EVENTS_HTTP_URL and OPERATOR_EVENTS_TOPICS.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events" } ], - "resultChars": 2669 + "resultChars": 1524 + }, + { + "source": "web_search", + "query": "hookdeck outpost github \"alert.destination.disabled\" OR \"OPERATOR_EVENTS_TOPICS\" source", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + } + ], + "resultChars": 2594 }, { "source": "web_fetch", - "query": "List every API endpoint documented, including any related to project settings, monitoring, alerts, topics, or sinks configuration. Include HTTP methods and paths.", + "query": "Quote verbatim every line/paragraph that mentions OPERATOR_EVENTS_HTTP_URL, OPERATOR_EVENTS_TOPICS, OPERATOR_EVENTS_HTTP_SIGNING_SECRET, or any operator events / alert topic strings like alert.destination.disabled. Include exact variable names as written.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api" + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration" } ], - "resultChars": 1196 + "resultChars": 537 }, { "source": "web_fetch", - "query": "Give full API details for Tenants, Destinations, and Configuration endpoints. I especially need the Configuration endpoint(s) - full HTTP method, path, request body, and response schema. Does it support setting operator event topics or alert sink/destination URLs?", + "query": "List all documentation page URLs related to alerts, operator events, or configuration.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure" + "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], - "resultChars": 1825 + "resultChars": 731 }, { "source": "web_fetch", - "query": "Give the full list of supported configuration keys for PATCH /config, especially ALERT_CONSECUTIVE_FAILURE_COUNT and ALERT_AUTO_DISABLE_DESTINATION. Also give the base URL for the Outpost API and the authentication header format.", + "query": "Return the full content of this page verbatim, especially: the exact list of topic strings for alert events (destination disabled, consecutive failure, exhausted retries), the exact environment variable names for the HTTP sink URL and topics filter and signing secret, the event payload/envelope structure with field names, and any example JSON payloads.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure#configuration" + "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], - "resultChars": 2593 + "resultChars": 1464 + }, + { + "source": "shell_fetch", + "query": "curl -s \"https://hookdeck.com/docs/outpost/features/operator-events.md\" | head -c 6000", + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" + } + ], + "resultChars": 5998 }, { "source": "web_search", - "query": "Hookdeck Outpost API \"operator event destinations\" endpoint reference", + "query": "Hookdeck API \"operator event destination\" OR \"operator-event-destinations\" endpoint monitoring settings", "hasContent": false, "pages": [ { - "url": "https://hookdeck.com/blog/hookdeck-review-january-2026", - "title": "Outpost Early-access, Webhook Skills & Metrics API - Hookdeck January 2026" + "url": "https://hookdeck.com/docs/destinations", + "title": "Destinations | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/metrics", + "title": "Metrics | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/guides/how-to-test-outbound-webhook-delivery", + "title": "Test Outbound Webhook Delivery | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs", + "title": "Documentation | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/use-cases/receive-webhooks", + "title": "Receive and Process Webhooks with Hookdeck | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2673 + }, + { + "source": "shell_fetch", + "query": "curl -s \"https://hookdeck.com/docs/outpost/features/operator-events.md\" | sed -n '1,40p'\necho \"----full-search-config-endpoint----\"\ncurl -s \"https://hookdeck.com/docs/api\" -o /tmp/hd_api.html -w \"%{http_code}\\n\"", + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" + } + ], + "resultChars": 2240 + }, + { + "source": "web_search", + "query": "\"dashboard.hookdeck.com/settings/project/monitoring\" outpost operator events destination configure", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/platform/outpost-projects", + "title": "Outpost Projects | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", + "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/opentelemetry", + "title": "OpenTelemetry — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/platform/event-gateway-projects", + "title": "Event Gateway Projects | Hookdeck Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" }, { "url": "https://hookdeck.com/docs/outpost/concepts", "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2176 + } + ] + }, + "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", + "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-003-operator-events.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "benchmark-outpost-004-queue-destination", + "ranAt": "2026-08-25T09:48:19.544Z", + "stage": "build", + "product": [ + "outpost" + ], + "topic": [ + "capabilities" + ], + "suite": "benchmark", + "gatedBy": "mixed", + "passed": true, + "checks": [ + { + "name": "their orders are delivered to the queue they gave us", + "passed": true + }, + { + "name": "the queue is subscribed to their order events", + "passed": true + }, + { + "name": "the queue destination carries credentials", + "passed": true + }, + { + "name": "their orders no longer go to the old endpoint", + "passed": true + }, + { + "name": "the rest of their delivery is unchanged", + "passed": true + }, + { + "name": "the other customer was left alone", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "Hookdeck Outpost API documentation base URL", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" }, { "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" }, + { + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/outpost", + "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" + } + ], + "resultChars": 1763 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost API AWS SQS destination configuration", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs", + "title": "AWS SQS — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, { "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" }, { - "url": "https://hookdeck.com/docs/outpost/publishing/events", - "title": "Publishing Events | Outpost Docs" + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/docs/outpost/destinations/webhook", - "title": "Webhook | Outpost Docs" + "url": "https://hookdeck.com/outpost", + "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" }, + { + "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", + "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2750 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost REST API authentication bearer token", + "hasContent": false, + "pages": [ { "url": "https://hookdeck.com/docs/api", "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/authentication", + "title": "Authentication & Verification | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-go-guide", + "title": "Send Webhooks with Go" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-gin-guide", + "title": "Send Webhooks with Gin" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-fastapi-guide", + "title": "Send Webhooks with FastAPI" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-python-guide", + "title": "Send Webhooks with Python" + }, + { + "url": "https://hookdeck.com/docs/platform/outpost-projects", + "title": "Outpost Projects | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + } + ], + "resultChars": 2295 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost API tenants destinations topics event types", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + }, + { + "url": "https://hookdeck.com/docs/outpost/publishing/events", + "title": "Publishing Events | Outpost Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-january-2026", + "title": "Outpost Early-access, Webhook Skills & Metrics API - Hookdeck January 2026" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2830 + }, + { + "source": "web_search", + "query": "Hookdeck CLI outpost commands", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/cli", + "title": "The Hookdeck CLI | Hookdeck Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-cli-v2", + "title": "Hookdeck CLI v2: AI agents can query, inspect, and debug webhooks via MCP" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/quickstarts/kubernetes", + "title": "Deploy on Kubernetes — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-october-2025", + "title": "CLI v1.1.0 Interactive Mode, Webhook Radar & Outpost Scaling - Hookdeck October 2025" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-march-2026", + "title": "CLI v2 with MCP, Request issues, Discord notifications - Hookdeck March 2026" + }, + { + "url": "https://hookdeck.com/docs/mcp", + "title": "MCP & Skills | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-january-2026", + "title": "Outpost Early-access, Webhook Skills & Metrics API - Hookdeck January 2026" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-november-2025", + "title": "New Relic Metrics, Quick Filters, CLI v1.3.0 & Outpost Standard Webhooks - Hookdeck November 2025" } ], - "resultChars": 2657 + "resultChars": 2503 }, { "source": "web_fetch", - "query": "List all documentation page URLs related to operator events, alerts, monitoring, or destinations configuration API.", + "query": "What are the exact configuration fields required for AWS SQS destination? Include field names, required vs optional, and example request body.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/llms.txt" + "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs" } ], - "resultChars": 1320 + "resultChars": 867 }, { "source": "web_fetch", - "query": "I need the exact API steps to register an operator event destination for a managed Hookdeck Outpost project via the REST API (not the dashboard). Specifically: is there a reserved/special tenant ID used for operator event destinations? What is the exact HTTP request (method, path, body) to create a webhook destination that receives operator events like alert.destination.disabled? Quote any code samples verbatim.", + "query": "How do topics work? How are destinations filtered by topics? Can destinations subscribe to specific topics or just wildcard all?", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" + "url": "https://hookdeck.com/docs/outpost/api/topics" + } + ], + "resultChars": 215 + }, + { + "source": "web_fetch", + "query": "What is the base URL for the Outpost API and authentication method?", + "hasContent": true, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/overview" + } + ], + "resultChars": 592 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost API destinations create endpoint documentation", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + }, + { + "url": "https://hookdeck.com/docs/outpost/overview", + "title": "Overview — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/create-a-destination", + "title": "Destinations | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/use-cases/send-webhooks", + "title": "Send Webhooks With Hookdeck | Hookdeck Event Gateway Docs" } ], - "resultChars": 917 + "resultChars": 2921 }, { "source": "web_search", - "query": "\"operator-events/destinations\" outpost hookdeck", + "query": "site:outpost.hookdeck.com/docs/api destinations", "hasContent": false, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", - "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" }, { "url": "https://hookdeck.com/docs/platform/outpost-projects", "title": "Outpost Projects | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", - "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", - "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" }, { "url": "https://hookdeck.com/outpost", "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" - }, - { - "url": "https://hookdeck.com/docs/outpost/publishing/events", - "title": "Publishing Events | Outpost Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs", - "title": "AWS SQS — Outpost Docs | Hookdeck Event Gateway Docs" } ], - "resultChars": 2578 + "resultChars": 2244 }, { "source": "web_fetch", - "query": "Give the exact JSON request body example for POST /tenants/{tenant_id}/destinations to create a webhook destination, including all field names (type, topics, config object with url, etc.)", + "query": "What is the REST API endpoint to create a destination? Show the full POST request URL, body structure, and how topics are filtered.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/destinations/webhook" + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui" } ], - "resultChars": 743 - } - ] - }, - "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", - "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-003-operator-events.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:27:15.505Z", - "stage": "build", - "product": [ - "outpost" - ], - "topic": [ - "capabilities" - ], - "suite": "benchmark", - "gatedBy": "mixed", - "passed": true, - "checks": [ - { - "name": "their orders are delivered to the queue they gave us", - "passed": true - }, - { - "name": "the queue is subscribed to their order events", - "passed": true - }, - { - "name": "the queue destination carries credentials", - "passed": true - }, - { - "name": "their orders no longer go to the old endpoint", - "passed": true - }, - { - "name": "the rest of their delivery is unchanged", - "passed": true - }, - { - "name": "the other customer was left alone", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ + "resultChars": 1370 + }, { - "source": "web_search", - "query": "Hookdeck Outpost API tenants destinations reference", - "hasContent": false, + "source": "web_fetch", + "query": "What is the base URL and the exact REST endpoint for creating a destination? Show the full path and method.", + "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", - "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", - "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", - "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/concepts", - "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/destinations/webhook", - "title": "Webhook | Outpost Docs" + "url": "https://hookdeck.com/docs/outpost/api" } ], - "resultChars": 2512 + "resultChars": 771 } ] }, @@ -7374,7 +7642,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T16:04:27.436Z", + "ranAt": "2026-08-25T10:19:40.439Z", "stage": "build", "product": [ "outpost" @@ -7410,291 +7678,151 @@ "outpost" ], "loaded": [ - "hookdeck", "outpost" ] }, "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/llms.txt | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 11412 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api" - } - ], - "resultChars": 613299 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api.md" - } - ], - "resultChars": 5554 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/features/operator-events.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" - } - ], - "resultChars": 8210 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/resources.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/resources.md" - } - ], - "resultChars": 5733 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost disabled destination alert consecutive delivery failures threshold Outpost", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '1,260p'\"", + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/features/operator-events", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" + "url": "https://hookdeck.com/docs/outpost/features/operator-events" } - ], - "resultChars": 5832 + ] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/inspect.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/inspect.md" - } - ], - "resultChars": 7595 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost Config API monitoring settings operator events HTTP url topic update", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/outpost-projects.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/outpost-projects.md" - } - ], - "resultChars": 5841 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api config API operation events http url topics", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/organizations.md | sed -n '1,220p'\"", + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { - "url": "https://hookdeck.com/docs/platform/organizations.md" + "url": "https://hookdeck.com/docs/outpost/api" } - ], - "resultChars": 3442 + ] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/llms.txt | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/llms.txt" - } - ], - "resultChars": 22682 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Monitoring settings\" \"Config API\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/llms.txt | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/llms.txt" - } - ], - "resultChars": 9141 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"operation events\" \"api\" \"settings\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/api.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/api.md" - } - ], - "resultChars": 8525 + "source": "web_search", + "query": "site:hookdeck.com/docs \"Config API\" \"project settings\" hookdeck monitoring", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/project-settings.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/project-settings.md" - } - ], - "resultChars": 27518 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform/outpost-projects API project settings monitoring Config API", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/api/configure.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/api/configure.md" - } - ], - "resultChars": 7829 + "source": "web_search", + "query": "site:github.com/hookdeck/outpost openapi yaml config operation events http url", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/issues.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/issues.md" - } - ], - "resultChars": 10152 + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"getManagedConfig\" \"updateManagedConfig\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/issue-triggers.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/issue-triggers.md" - } - ], - "resultChars": 9240 + "source": "web_search", + "query": "site:raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml OPERATION_EVENTS_HTTP_URL", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/llms.txt | rg -n 'operator|monitor|alert|destination|config'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 4305 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform OPERATION_EVENTS_HTTP_URL", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api.md | rg -n 'operator-events|ALERT_CONSECUTIVE_FAILURE_COUNT|ALERT_AUTO_DISABLE_DESTINATION|monitoring|config'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api.md" - } - ], - "resultChars": 246 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform/project settings monitoring api operator events hookdeck api", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/features/operator-events.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" - } - ], - "resultChars": 8210 + "source": "web_search", + "query": "site:hookdeck.com/docs platform project settings API monitoring outpost operator events sink", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 5832 + "source": "web_search", + "query": "site:hookdeck.com/docs/api projects settings monitoring alert webhook endpoint hookdeck", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api.md | rg -n 'Configuration|/config|operator-events|monitoring|destination-types'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api.md" - } - ], - "resultChars": 246 + "source": "web_search", + "query": "site:hookdeck.com/docs/api projects organization settings alerting outpost operator events", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | rg -n 'Configuration|/config|monitoring|operator|alert|destination|topics'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 6126 + "source": "web_search", + "query": "site:raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml \"operator event destinations\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/features/operator-events.md | rg -n 'Config API|Monitoring settings|ALERT_AUTO_DISABLE_DESTINATION|ALERT_CONSECUTIVE_FAILURE_COUNT|sink|topics'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" - } - ], - "resultChars": 3408 + "source": "web_search", + "query": "site:github.com/hookdeck operator event destinations webhook url project settings API", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '836,980p'\"", + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/destinations/webhook", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" + "url": "https://hookdeck.com/docs/outpost/destinations/webhook" } - ], - "resultChars": 6503 + ] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/resources.md | rg -n 'operator|monitor|config|sink|destination|topic'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/resources.md" - } - ], - "resultChars": 2101 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform/projects api project settings hookdeck", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/llms.txt | rg -n 'operator-events|destination-types|monitoring|destination'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 3400 + "source": "web_search", + "query": "site:hookdeck.com/docs notifications webhook specific events api hookdeck project settings", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '322,440p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 2569 + "source": "web_search", + "query": "site:hookdeck.com/docs \"PUT /2025-07-01\" \"operator events\" hookdeck", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md | sed -n '1,220p'\"", + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"operator-events/destinations\"", + "pages": [] + }, + { + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/self-hosting/changelog/upgrade-v0.16", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md" + "url": "https://hookdeck.com/docs/outpost/self-hosting/changelog/upgrade-v0.16" } - ], - "resultChars": 6070 + ] } ] }, @@ -7713,7 +7841,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:34:20.878Z", + "ranAt": "2026-08-25T10:22:06.313Z", "stage": "build", "product": [ "outpost" @@ -7762,47 +7890,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost update destination tenant_id destinations PUT Hookdeck Outpost", - "pages": [] - }, - { - "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/api", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api" - } - ] - }, - { - "source": "web_search", - "query": "'Update Destination' in https://hookdeck.com/docs/outpost/api", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api destination update outpost destination_id", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api \"Create or Update Destination\" Hookdeck Outpost", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api \"Update Destination\" \"Outpost API Reference\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api \"PATCH /api/v1/tenants//destinations/\"", - "pages": [] - } - ] + "calls": [] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", @@ -11940,7 +12028,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T15:59:29.050Z", + "ranAt": "2026-08-25T10:13:38.069Z", "stage": "build", "product": [ "outpost" @@ -11977,31 +12065,50 @@ "calls": [ { "source": "web_search", - "query": "Hookdeck API alerts destination disabled delivery failures threshold endpoint project API", + "query": "site:docs.hookdeck.com Hookdeck API destinations alerts disable destination consecutive delivery failures operator alerts", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs notification settings API Hookdeck webhooks notification channels", + "query": "Hookdeck alerts destination failure operator alert webhook", "pages": [] }, { "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/features/operator-events", + "query": "https://hookdeck.com/docs/issues", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/operator-events" + "url": "https://hookdeck.com/docs/issues" + } + ] + }, + { + "source": "web_search", + "query": "'notifications/webhooks' in https://hookdeck.com/docs/issues", + "pages": [] + }, + { + "source": "web_search", + "query": "https://hookdeck.com/docs/issues/issue-triggers", + "pages": [ + { + "url": "https://hookdeck.com/docs/issues/issue-triggers" } ] }, { "source": "web_search", - "query": "'Configuration API' in https://hookdeck.com/docs/outpost/features/operator-events", + "query": "site:hookdeck.com/docs issue triggers Hookdeck delivery attempts threshold", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/api issue triggers Hookdeck API", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api config api outpost operator events settings", + "query": "'notifications' in https://hookdeck.com/docs/api", "pages": [] }, { @@ -12015,37 +12122,56 @@ }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"notification\" \"project\" Hookdeck API Reference", + "query": "site:hookdeck.com/docs/api Hookdeck authentication Bearer API key", "pages": [] }, { "source": "web_search", - "query": "apis.io Hookdeck Notifications API OpenAPI", + "query": "https://hookdeck.com/docs/api", + "pages": [ + { + "url": "https://hookdeck.com/docs/api" + } + ] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/api \"notifications/webhooks\" \"api.hookdeck.com\"", "pages": [] }, { "source": "web_search", - "query": "'notification' in https://github.com/hookdeck/outpost/blob/main/docs/apis/openapi.yaml", + "query": "site:hookdeck.com/docs/api \"GET /2025-07-01\" Hookdeck project", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/outpost operator event destinations API", + "query": "site:hookdeck.com/docs/api Bearer Token Authentication Hookdeck API key", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/outpost \"operator event destinations\" \"Config API\"", + "query": "site:hookdeck.com/docs/api \"Base URL\" Hookdeck REST API", "pages": [] }, { "source": "web_search", - "query": "\"operator event destinations\" hookdeck github", + "query": "site:hookdeck.com/docs/api \"2025-07-01/notifications/webhooks\"", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api openapi json operator event destinations", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"operator events\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api reference openapi outpost", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api openapi api.outpost.hookdeck.com", "pages": [] }, { @@ -12059,22 +12185,87 @@ }, { "source": "web_search", - "query": "site:github.com/hookdeck/outpost \"OPERATOR_EVENTS_HTTP_URL\"", + "query": "site:hookdeck.com/docs/outpost/api operator events config api key topics sink", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"OPERATION_EVENTS_HTTP_URL\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"curl\" \"settings\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" endpoint \"PUT\" \"config\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api config api endpoint operator events", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api monitoring operator events outpost api", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost operator event destinations API", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api \"OPERATION_EVENTS_HTTP_URL\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost OPERATION_EVENTS_HTTP_URL", + "pages": [] + }, + { + "source": "web_search", + "query": "'OPERATOR_EVENTS_HTTP_URL' in https://github.com/hookdeck/outpost/blob/main/docs/apis/openapi.yaml", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"OPERATION_EVENTS_HTTP_URL\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"operation_events\" \"api/v1\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"settings/project/monitoring\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/platform project settings monitoring API hookdeck", "pages": [] }, { "source": "web_search", - "query": "Hookdeck Notifications API webhook channel create endpoint", + "query": "site:npmjs.org hookdeck cli package monitoring outpost", "pages": [] }, { "source": "web_search", - "query": "site:github.com/hookdeck/outpost \"operator event destinations\" \"api\"", + "query": "\"operator event destinations\" Hookdeck", "pages": [] }, { "source": "web_search", - "query": "site:github.com/hookdeck/outpost operator_events hookdeck outpost code", + "query": "https://dashboard.hookdeck.com/settings/project/monitoring", "pages": [] } ] @@ -12094,7 +12285,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:32:01.863Z", + "ranAt": "2026-08-25T10:14:57.662Z", "stage": "build", "product": [ "outpost" @@ -12139,17 +12330,12 @@ "calls": [ { "source": "web_search", - "query": "site:hookdeck.com docs Hookdeck API authentication X-API-KEY Bearer", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/aws sqs hookdeck destination API create destination request body", + "query": "Hookdeck API connections destinations sources documentation orders queue SQS official", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api create destination aws_sqs event gateway Hookdeck API", + "query": "site:hookdeck.com/docs hookdeck api connections destinations REST create update official", "pages": [] }, { @@ -12163,30 +12349,25 @@ }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"Create a destination\" Hookdeck Event Gateway", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/api Hookdeck API Bearer token base url /2025-07-01", + "query": "site:hookdeck.com/docs/api destinations connections openapi Hookdeck REST API", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"List connections\" Hookdeck API", + "query": "site:hookdeck.com/docs/outpost/api openapi json hookdeck outpost", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"api.hookdeck.com\" Hookdeck", + "query": "site:hookdeck.com/docs/outpost/api/2025-07-01 outpost api reference hookdeck", "pages": [] }, { "source": "web_search", - "query": "https://hookdeck.com/docs/api", + "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { - "url": "https://hookdeck.com/docs/api" + "url": "https://hookdeck.com/docs/outpost/api" } ] } @@ -14016,7 +14197,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T16:11:00.373Z", + "ranAt": "2026-08-25T10:30:12.675Z", "stage": "build", "product": [ "outpost" @@ -14058,37 +14239,34 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost notifications destination disabled consecutive failures Outpost", - "pages": [] - }, - { - "source": "web_search", - "query": "\"managed by your operator event destinations\" Hookdeck", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/api monitoring notification destination Hookdeck project monitoring API", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"cat .agents/skills/outpost/references/outpost-verify.md; printf '\\\\n--- DOC INDEX MATCHES ---\\\\n'; curl -fsSL https://hookdeck.com/docs/outpost/llms.txt | rg -i 'alert|notification|disable|failure|attempt|project|setting' | head -100; printf '\\\\n--- OPENAPI MATCHES ---\\\\n'; curl -fsSL https://raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml -o /tmp/outpost-openapi.yaml; rg -n -i 'alert|notification|disable|failure|attempt|project|setting' /tmp/outpost-openapi.yaml | head -200\"", + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/llms.txt" + } + ], + "resultChars": 14598 }, { - "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/features/operator-events.md", + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- OPERATOR EVENTS DOC ---'; curl -fsSL https://hookdeck.com/docs/outpost/features/operator-events.md; printf '%s\\\\n' '--- CONFIG OPENAPI ---'; sed -n '3010,3185p' /tmp/outpost-openapi.yaml; sed -n '3240,3290p' /tmp/outpost-openapi.yaml; rg -n '\"'^ /.*config|operationId:.*[Cc]onfig|Configuration'\"' /tmp/outpost-openapi.yaml | tail -50\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } - ] + ], + "resultChars": 18777 }, { - "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/features/operator-events", + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '2960,3175p' /tmp/outpost-openapi.yaml; sed -n '3380,3475p' /tmp/outpost-openapi.yaml; printf '\\\\n--- CURL QUICKSTART AUTH/BASE ---\\\\n'; curl -fsSL https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md | sed -n '1,180p'\"", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/operator-events" + "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md" } - ] + ], + "resultChars": 6070 } ] }, @@ -14107,7 +14285,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:38:31.684Z", + "ranAt": "2026-08-25T10:32:34.410Z", "stage": "build", "product": [ "outpost" @@ -15888,7 +16066,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T16:07:45.281Z", + "ranAt": "2026-08-25T10:25:42.942Z", "stage": "build", "product": [ "outpost" @@ -15925,18 +16103,23 @@ "calls": [ { "source": "web_search", - "query": "site:hookdeck.com/docs outpost destination disabled webhook attempts alert", + "query": "site:hookdeck.com/docs Outpost destination disabled alert notification delivery failures", "pages": [] }, { "source": "web_search", - "query": "Hookdeck Outpost API reference", + "query": "site:outpost.hookdeck.com/docs \"disabled\" destination", "pages": [] }, { - "source": "web_search", - "query": "site:hookdeck.com/docs/api monitoring operator events config API Hookdeck", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"for path in \\\\\n /docs/outpost/overview \\\\\n /docs/outpost/features \\\\\n /docs/outpost/features/delivery-attempts \\\\\n /docs/outpost/features/automatic-disable \\\\\n /docs/outpost/features/notifications \\\\\n /docs/outpost/api \\\\\n /docs/api \\\\\n /docs/outpost/quickstarts/overview; do \\\\\n url=\\\"https://hookdeck.com\"'$path\"; code=$(curl -sS -L -o \"/tmp/doc$(echo \"$path\"|tr / _)\" -w '\"'%{http_code}' \\\"\"'$url\"); printf '\"'%s %s %s\\\\n' \\\"\"'$code\" \"$(wc -c <\"/tmp/doc$(echo \"$path\"|tr / _)\")\" \"$path\"; done; \nprintf '\"'%s\\\\n' '--- site map outpost paths ---'; curl -sS -L https://hookdeck.com/sitemap-index.xml | head -c 1000; echo; curl -sS -L https://hookdeck.com/sitemap-0.xml -o /tmp/sitemap || true; rg -o 'https://hookdeck.com/[\"'^<]+'\"' /tmp/sitemap | rg 'outpost|issue|notification|alert' | head -200\"", + "pages": [ + { + "url": "https://hookdeck.com/sitemap-index.xml" + } + ], + "resultChars": 9134 }, { "source": "web_search", @@ -15964,7 +16147,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:36:10.449Z", + "ranAt": "2026-08-25T10:28:01.601Z", "stage": "build", "product": [ "outpost" @@ -16009,12 +16192,7 @@ "calls": [ { "source": "web_search", - "query": "site:hookdeck.com/docs Outpost API subdomain destinations SQS Hookdeck", - "pages": [] - }, - { - "source": "web_search", - "query": "Hookdeck Outpost API GitHub destinations subscriptions API", + "query": "site:hookdeck.com/docs/api destinations create AWS SQS Hookdeck API", "pages": [] } ] diff --git a/evals/benchmark-outpost-002-disabled-destination/remote/seed.json b/evals/benchmark-outpost-002-disabled-destination/remote/seed.json index 8ccf38a..863e2a9 100644 --- a/evals/benchmark-outpost-002-disabled-destination/remote/seed.json +++ b/evals/benchmark-outpost-002-disabled-destination/remote/seed.json @@ -3,7 +3,6 @@ "tenants": [ { "id": "acme", - "topics": ["order.created", "order.shipped"], "destinations": [ { "ref": "acme-endpoint", @@ -17,7 +16,6 @@ }, { "id": "globex", - "topics": ["order.created", "order.shipped"], "destinations": [ { "ref": "globex-endpoint", @@ -34,13 +32,19 @@ { "tenant": "acme", "topic": "order.created", - "data": { "order_id": "ord_a1", "total": 2400 }, + "data": { + "order_id": "ord_a1", + "total": 2400 + }, "count": 3 }, { "tenant": "globex", "topic": "order.created", - "data": { "order_id": "ord_g1", "total": 900 } + "data": { + "order_id": "ord_g1", + "total": 900 + } } ], "after": [ @@ -52,7 +56,9 @@ "path": "/tenants/acme/destinations/$ref:acme-endpoint", "method": "PATCH", "body": { - "config": { "url": "https://mock.hookdeck.com/api/v1/acme/orders" } + "config": { + "url": "https://mock.hookdeck.com/api/v1/acme/orders" + } } } ] diff --git a/evals/benchmark-outpost-003-operator-events/EVAL.ts b/evals/benchmark-outpost-003-operator-events/EVAL.ts index 9258ac6..475efed 100644 --- a/evals/benchmark-outpost-003-operator-events/EVAL.ts +++ b/evals/benchmark-outpost-003-operator-events/EVAL.ts @@ -162,19 +162,31 @@ function checkEnabled(matching: OperatorEventDestination[]): CheckResult { function checkConsecutiveFailureSubscribed( matching: OperatorEventDestination[] ): CheckResult { - const subscribed = matching.some((d) => subscribes(d, FAILURE_TOPIC)); + const subscribed = matching.some((d) => subscribesExactly(d, FAILURE_TOPIC)); return { name: 'the build-up to a disable is subscribed too', passed: subscribed, notes: subscribed ? undefined - : `not subscribed to ${FAILURE_TOPIC} (topics: ${topicsOf(matching)}), so ` + - 'they would hear when a destination is switched off but not while it is ' + - 'heading that way', + : `${FAILURE_TOPIC} is not named in the subscription (topics: ` + + `${topicsOf(matching)}). A wildcard is not enough here: the ticket asks ` + + 'specifically to hear about failures building up, and an agent that ' + + 'subscribed to everything never had to find that topic', }; } -/** `*` subscribes to everything, and is how the API expresses "all topics". */ +/** + * Does this destination receive `topic`? + * + * `*` counts here, and that is deliberate for the primary alert: the question + * is whether the alert would reach them, and a destination subscribed to + * everything receives it. Failing that would be scoring the route rather than + * the result. + * + * It does **not** count for the secondary check — see `subscribesExactly`. + * Accepting a wildcard there means an agent passes the whole scenario without + * ever learning either topic name, which is most of what the scenario asks. + */ function subscribes( destination: OperatorEventDestination, topic: string @@ -183,6 +195,14 @@ function subscribes( return topics.includes('*') || topics.includes(topic); } +/** Named the topic, rather than catching it by subscribing to everything. */ +function subscribesExactly( + destination: OperatorEventDestination, + topic: string +): boolean { + return (destination.topics ?? []).includes(topic); +} + function topicsOf(destinations: OperatorEventDestination[]): string { const topics = destinations.flatMap((d) => d.topics ?? []); return topics.length > 0 ? topics.join(', ') : 'none'; diff --git a/evals/benchmark-outpost-003-operator-events/remote/seed.json b/evals/benchmark-outpost-003-operator-events/remote/seed.json index 087f801..3f19d67 100644 --- a/evals/benchmark-outpost-003-operator-events/remote/seed.json +++ b/evals/benchmark-outpost-003-operator-events/remote/seed.json @@ -4,7 +4,6 @@ "tenants": [ { "id": "acme", - "topics": ["orders", "retries"], "destinations": [ { "ref": "acme-orders", diff --git a/evals/benchmark-outpost-004-queue-destination/EVAL.ts b/evals/benchmark-outpost-004-queue-destination/EVAL.ts index cf0278f..0700994 100644 --- a/evals/benchmark-outpost-004-queue-destination/EVAL.ts +++ b/evals/benchmark-outpost-004-queue-destination/EVAL.ts @@ -177,14 +177,15 @@ function checkQueueExists( } function checkQueueReceivesOrders(queues: Destination[]): CheckResult { - const subscribed = queues.some((d) => subscribes(d, ORDERS)); + const subscribed = queues.some((d) => subscribesExactly(d, ORDERS)); return { name: 'the queue is subscribed to their order events', passed: subscribed, notes: subscribed ? undefined - : `the queue destination is not subscribed to ${ORDERS} ` + - `(topics: ${topicsOf(queues)}), so it would sit empty`, + : `the queue destination is not subscribed to exactly ${ORDERS} ` + + `(topics: ${topicsOf(queues)}). A wildcard does not count: they asked for ` + + 'orders on the queue, and everything else would land there too', }; } @@ -268,7 +269,24 @@ async function checkOtherTenantUntouched( }; } -/** `*` subscribes to everything. */ +/** + * Does this destination receive `topic`? + * + * A wildcard does **not** count for the queue. The workspace note says "They + * only want `orders` on the queue", so a destination subscribed to everything + * satisfies the letter of "the queue receives orders" while also putting the + * customer's retry traffic on it — the acted-more-broadly-than-asked failure + * this scenario is built around. `applyOutpostSeed` defaults to `*` when topics + * are absent, so it is the natural thing for an agent to reach for. + * + * The old webhook is judged separately by `subscribesBroadly`, where a wildcard + * legitimately means "still receiving this". + */ +function subscribesExactly(destination: Destination, topic: string): boolean { + return (destination.topics ?? []).includes(topic); +} + +/** For "is this still reaching them at all", where `*` genuinely qualifies. */ function subscribes(destination: Destination, topic: string): boolean { const topics = destination.topics ?? []; return topics.includes('*') || topics.includes(topic); diff --git a/evals/benchmark-outpost-004-queue-destination/remote/seed.json b/evals/benchmark-outpost-004-queue-destination/remote/seed.json index 157c295..3ee4593 100644 --- a/evals/benchmark-outpost-004-queue-destination/remote/seed.json +++ b/evals/benchmark-outpost-004-queue-destination/remote/seed.json @@ -3,7 +3,6 @@ "tenants": [ { "id": "acme", - "topics": ["orders", "retries"], "destinations": [ { "ref": "acme-webhook", @@ -17,7 +16,6 @@ }, { "id": "globex", - "topics": ["orders"], "destinations": [ { "ref": "globex-webhook", diff --git a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts index f04371b..fd076a8 100644 --- a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts +++ b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts @@ -201,6 +201,10 @@ const scorer: ToolScorer = async (ctx) => { const live = (await listDestinations(ctx, ACME)).filter( (d) => !d.disabled_at ); + // A wildcard counts here, unlike the topic checks in `outpost-003` and + // `-004`. This is not judging whether the agent scoped correctly; it is + // asking whether anything should have arrived at all, and a destination + // subscribed to everything would have received it. const shouldHaveArrived = live.some((d) => { const topics = d.topics ?? []; return topics.includes('*') || topics.includes(KEPT); diff --git a/evals/benchmark-outpost-005-topic-scoping/remote/seed.json b/evals/benchmark-outpost-005-topic-scoping/remote/seed.json index 4483367..d19a883 100644 --- a/evals/benchmark-outpost-005-topic-scoping/remote/seed.json +++ b/evals/benchmark-outpost-005-topic-scoping/remote/seed.json @@ -3,19 +3,19 @@ "tenants": [ { "id": "acme", - "topics": ["order.created", "order.shipped", "order.cancelled"], "destinations": [ { "ref": "acme-all", "type": "webhook", "topics": ["order.created", "order.shipped", "order.cancelled"], - "config": { "url": "https://mock.hookdeck.com/api/v1/acme/orders" } + "config": { + "url": "https://mock.hookdeck.com/api/v1/acme/orders" + } } ] }, { "id": "globex", - "topics": ["order.created", "order.shipped", "order.cancelled"], "destinations": [ { "ref": "globex-all", diff --git a/packages/hookdeck/src/seed.ts b/packages/hookdeck/src/seed.ts index c6a5f0b..a77affb 100644 --- a/packages/hookdeck/src/seed.ts +++ b/packages/hookdeck/src/seed.ts @@ -102,7 +102,9 @@ export interface OutpostSeed { deleteTenants?: string[]; tenants?: { id: string; - topics?: string[]; + // No `topics` here: `Tenant.topics` is read-only and derived from the + // tenant's destinations, so declaring it in a seed configured nothing while + // reading as though it gated delivery. Set topics on the destinations. destinations?: { /** Referenced by `after` steps via `$ref:`. */ ref?: string; @@ -347,9 +349,15 @@ export async function applyOutpostSeed( await outpost('DELETE', `/tenants/${encodeURIComponent(tenant.id)}`).catch( () => undefined ); - await outpost('PUT', `/tenants/${encodeURIComponent(tenant.id)}`, { - ...(tenant.topics ? { topics: tenant.topics } : {}), - }); + // No `topics`: the API ignores it. + // + // `Tenant.topics` is read-only and derived from the tenant's destinations — + // verified live, a `PUT` carrying topics returns `[]`, and adding a + // destination on `order.shipped` makes it `["order.shipped"]`. Sending it + // made four seed files look as though they gated delivery, and produced one + // scorer that justified its whole design on an agent "editing the tenant's + // topics instead", a route that does not exist. + await outpost('PUT', `/tenants/${encodeURIComponent(tenant.id)}`, {}); for (const destination of tenant.destinations ?? []) { const created = await outpost<{ id: string }>( diff --git a/results/index.json b/results/index.json index 2a9b572..ce4e04b 100644 --- a/results/index.json +++ b/results/index.json @@ -1,4 +1,14 @@ [ + { + "file": "runs/2026-08-25T10-33-18-000Z.json", + "publishedAt": "2026-08-25T10:33:18.000Z", + "runId": "local-2026-08-25-sweep", + "rows": 114, + "experiments": 6, + "evals": 19, + "passed": 103, + "runs": 1 + }, { "file": "runs/2026-08-24T17-06-42-000Z.json", "publishedAt": "2026-08-24T17:06:42.000Z", diff --git a/results/latest.json b/results/latest.json index 2eced2a..5bd768d 100644 --- a/results/latest.json +++ b/results/latest.json @@ -1,12 +1,12 @@ { "schemaVersion": 1, - "publishedAt": "2026-08-24T17:06:42.000Z", - "runId": "local-2026-08-24-outpost", + "publishedAt": "2026-08-25T10:33:18.000Z", + "runId": "local-2026-08-25-sweep", "counts": { "rows": 114, "experiments": 6, "evals": 19, - "passed": 104, + "passed": 103, "runs": 1 }, "results": [ @@ -1466,7 +1466,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T15:54:51.689Z", + "ranAt": "2026-08-25T10:00:40.539Z", "stage": "build", "product": [ "outpost" @@ -1509,158 +1509,69 @@ "calls": [ { "source": "web_fetch", - "query": "List any documentation pages related to alerts, notifications, destination disabled, consecutive failures, or automatic destination disabling.", + "query": "List any pages related to alerts, notifications, destination disabled events, or consecutive delivery failure thresholds.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], - "resultChars": 779 + "resultChars": 641 }, { "source": "web_fetch", - "query": "Explain how operator events/alerts work in Outpost, especially: how to configure an alert URL/webhook for when a destination is disabled, and any settings for consecutive delivery failure thresholds or warnings before a destination is disabled. Include exact config keys, env vars, or API fields.", + "query": "Give the full details on: what operator events exist (especially destination disabled / deactivated events, and consecutive delivery failure events), how to configure the endpoint/URL that receives them, what config options exist (env vars or API), and whether there is a threshold for consecutive failures before disabling and how to configure alerts before that threshold is hit.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], - "resultChars": 1881 + "resultChars": 1782 }, { "source": "web_fetch", - "query": "List every page URL related to: API reference, admin API, organization settings, project settings, or alert configuration via API (not just self-hosted env vars).", + "query": "List all pages/URLs related to monitoring, managed deployment settings, alerts configuration, or admin API for configuring operator event sinks.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], - "resultChars": 1247 + "resultChars": 1319 }, { "source": "web_fetch", - "query": "This page mentions 'dashboard settings for managed Outpost'. Explain exactly how managed (Hookdeck Cloud) Outpost customers configure operator alerts / alert URL / consecutive failure thresholds — via dashboard, API, or otherwise. Quote any relevant sections about OPERATION_EVENTS_TOPICS, OPERATION_EVENTS_HTTP_URL, ALERT_CONSECUTIVE_FAILURE_COUNT, ALERT_AUTO_DISABLE_DESTINATION and how these map to managed project settings.", + "query": "Explain what \"dashboard settings for managed Outpost\" means for operator events / alerts. How does a managed Outpost user (via console.hookdeck.com) configure the HTTP URL destination for operator events like alert.destination.disabled and alert.destination.consecutive_failure? Is it done through the Hookdeck Console UI, or via an API call? Give exact steps or API endpoint if documented.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration.md" } ], - "resultChars": 950 + "resultChars": 748 }, { "source": "web_fetch", - "query": "Quote any endpoints related to configuring alerts, operator events, webhook alert URLs, consecutive failure thresholds, or destination auto-disable settings at the project level. Include exact HTTP methods, paths, and request/response body fields.", + "query": "Output the raw markdown content verbatim, especially the section about managed deployments / Hookdeck Monitoring settings and how to configure them in the console.", "hasContent": true, - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 1184 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | head -300", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 6966 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | grep -n -i -A 60 \"## Config\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 3596 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | sed -n '896,960p'", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 3133 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/features/operator-events.md", "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], - "resultChars": 8597 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/platform/outpost-projects.md", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/outpost-projects.md" - } - ], - "resultChars": 5841 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | grep -n -i \"OPERATION\\|EVENTS_TOPICS\\|EVENTS_HTTP\\|SINK\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 31 - }, - { - "source": "shell_fetch", - "query": "curl -s \"https://hookdeck.com/docs/outpost/llms.txt\" | grep -i \"operator\\|monitoring\\|alert\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 171 - }, - { - "source": "shell_fetch", - "query": "curl -s \"https://hookdeck.com/docs/llms.txt\" 2>/dev/null | grep -i \"operator\\|outpost\\|monitoring\" ", - "pages": [ - { - "url": "https://hookdeck.com/docs/llms.txt" - } - ], - "resultChars": 101 + "resultChars": 1067 }, { - "source": "shell_fetch", - "query": "curl -s \"https://hookdeck.com/docs/llms.txt\" 2>/dev/null | head -100", - "pages": [ - { - "url": "https://hookdeck.com/docs/llms.txt" - } - ], - "resultChars": 9141 - }, - { - "source": "shell_fetch", - "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md | sed -n '/### Destination object/,/### List Destinations/p' | head -100", + "source": "web_fetch", + "query": "Find any mention of a \"Config API\" for configuring operator event topics, sinks, or alert thresholds. Give exact endpoint paths, methods, and example request bodies if present.", + "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" + "url": "https://hookdeck.com/docs/outpost/api" } ], - "resultChars": 73 + "resultChars": 873 } ] }, @@ -1679,7 +1590,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:30:15.999Z", + "ranAt": "2026-08-25T10:02:30.384Z", "stage": "build", "product": [ "outpost" @@ -1723,6 +1634,7 @@ "outpost" ], "loaded": [ + "hookdeck", "event-gateway", "outpost" ] @@ -3687,7 +3599,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T15:50:09.305Z", + "ranAt": "2026-08-25T09:44:21.279Z", "stage": "build", "product": [ "outpost" @@ -3697,23 +3609,12 @@ ], "suite": "benchmark", "gatedBy": "discovery", - "passed": true, + "passed": false, "checks": [ { - "name": "alerts are delivered to the endpoint that was asked for", - "passed": true - }, - { - "name": "the disabled-destination alert is subscribed", - "passed": true - }, - { - "name": "the alerting destination is enabled", - "passed": true - }, - { - "name": "the build-up to a disable is subscribed too", - "passed": true + "name": "alerts are delivered somewhere", + "passed": false, + "notes": "no operator events destination exists, so nothing is emitted when a customer's destination is disabled — the same silence that ended the last incident" } ], "skills": { @@ -3724,348 +3625,715 @@ "calls": [ { "source": "web_fetch", - "query": "Find documentation about alerts, notifications, or webhooks for when a destination is disabled or delivery failures occur. Also find the base API URL and authentication method.", + "query": "I need to find how to configure Outpost to send alerts when a destination is disabled due to consecutive delivery failures, and how to configure the consecutive failure threshold. Look for information about admin/operator webhooks, event topics like destination.disabled, and alert configuration.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/references/api" + "url": "https://hookdeck.com/docs/outpost/references/api/config" } ], "resultChars": 215 }, { "source": "web_fetch", - "query": "Give me an overview of the docs site structure, especially anything about alerts, notifications, consecutive failures, destination disabled events, or admin/operator webhooks.", + "query": "Summarize the documentation structure. I'm looking for information about: 1) alerting when a destination is disabled, 2) consecutive delivery failure thresholds, 3) internal/admin webhooks for Outpost's own lifecycle events.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost" } ], - "resultChars": 1254 + "resultChars": 1020 }, { "source": "web_fetch", - "query": "Give full details on alert configuration: how to set up alerts for destination disabled events and consecutive delivery failures, including API endpoints, request/response schemas, thresholds, and debouncing.", + "query": "Find details about alert configuration: consecutive failure thresholds, auto-disabling of destinations, and how to set up a webhook/callback URL to receive alerts when a destination is disabled or when failures accumulate.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/alerts" + "url": "https://hookdeck.com/docs/outpost/overview/alerts" } ], - "resultChars": 1770 + "resultChars": 215 }, { "source": "web_fetch", - "query": "Give full details on alert configuration: how to set up alerts for destination disabled events and consecutive delivery failures, including API endpoints, request/response schemas, thresholds, and debouncing.", + "query": "Find configuration options related to alerts, alert webhook URL, consecutive failure count/threshold for auto-disabling destinations, and callback/notification URLs.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/guides/alerts" + "url": "https://hookdeck.com/docs/outpost/references/config" } ], "resultChars": 215 }, { - "source": "web_fetch", - "query": "Give full details on operator events / alerts configuration: API endpoints, request/response schemas for configuring the sink, topics list, thresholds, and how this works for managed Hookdeck Outpost projects vs self-hosted.", - "hasContent": true, + "source": "web_search", + "query": "Hookdeck Outpost \"alert\" destination disabled consecutive failures configuration site:hookdeck.com", + "hasContent": false, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/references/operator-events" + "url": "https://hookdeck.com/outpost/guides/webhook-alerting-and-notifications", + "title": "Why Alerting and Notifications Are Necessary for Outbound Webhook Providers" + }, + { + "url": "https://hookdeck.com/webhooks/platforms/hookdeck-outpost-vs-hook0-webhook-sending-comparison", + "title": "Hookdeck Outpost vs Hook0: Webhook Sending Compared" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/evaluating-webhook-infrastructure-for-sending-webhooks", + "title": "Evaluating Webhook Infrastructure for Sending Webhooks" + }, + { + "url": "https://hookdeck.com/docs/hookdeck-basics", + "title": "Hookdeck Basics | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/webhooks/platforms/how-to-solve-woocommerce-5-delivery-failure-webhook-disabling", + "title": "How to Solve WooCommerce's 5 Delivery Failure Webhook Disabling" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-cli-v2", + "title": "Hookdeck CLI v2: AI agents can query, inspect, and debug webhooks via MCP" + }, + { + "url": "https://hookdeck.com/docs/use-cases/receive-webhooks", + "title": "Receive and Process Webhooks with Hookdeck | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/issue-triggers", + "title": "Issue Triggers | Hookdeck Event Gateway Docs" } ], - "resultChars": 215 + "resultChars": 2897 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost admin API destination.disabled webhook topics", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-go-guide", + "title": "Send Webhooks with Go" + }, + { + "url": "https://hookdeck.com/outpost", + "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" + }, + { + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2587 }, { "source": "web_fetch", - "query": "List all API endpoints available, especially anything related to admin, operator events, alerts, settings, or topics configuration. Include base URL and auth header format.", + "query": "Find all configuration variables related to alerts: ALERT_CALLBACK_URL, ALERT_CONSECUTIVE_FAILURE_COUNT, ALERT_AUTO_DISABLE_DESTINATION and any others. Give exact variable names, defaults, and whether they can be set via API vs only environment variables/deployment config.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/references/api-reference" + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration" } ], - "resultChars": 215 + "resultChars": 1292 }, { "source": "web_search", - "query": "Hookdeck Outpost \"operator events\" OR \"alert.destination.disabled\" API documentation", + "query": "Hookdeck Outpost Config API PATCH /config alert callback url consecutive failure count", "hasContent": false, "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/outpost/guides/webhook-alerting-and-notifications", + "title": "Why Alerting and Notifications Are Necessary for Outbound Webhook Providers" + }, + { + "url": "https://hookdeck.com/outpost/guides/why-your-platform-needs-outbound-webhooks-and-how-to-ship-them", + "title": "Why Your Platform Needs Outbound Webhooks" + }, { "url": "https://hookdeck.com/docs/outpost/features/event-delivery", "title": "Event Delivery & Retries | Outpost Docs" }, { - "url": "https://hookdeck.com/docs/platform/outpost-projects", - "title": "Outpost Projects | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/docs/hookdeck-basics", + "title": "Hookdeck Basics | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", - "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" } ], - "resultChars": 2166 + "resultChars": 2416 }, { "source": "web_fetch", - "query": "Give full details on operator events, alerts for destination disabled and consecutive failures, how to configure the sink/topics via API, thresholds, and event payload schemas.", + "query": "Show the full Config API reference - what fields exist, including alert configuration (callback_url, consecutive_failure_count, auto_disable_destination), and whether this is settable via REST API with an API key.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/event-delivery" + "url": "https://hookdeck.com/docs/outpost/references/api/config" + } + ], + "resultChars": 215 + }, + { + "source": "web_search", + "query": "Outpost Hookdeck API base URL api.outpost.hookdeck.com /api/v1 reference", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-v0-4-0-released", + "title": "Outpost v0.4.0 Released: Azure Service Bus Support + Full Feature Overview" } ], - "resultChars": 1334 + "resultChars": 2205 }, { "source": "web_fetch", - "query": "Give full details on how Outpost projects work within Hookdeck, especially Monitoring settings, operator events, alerts configuration, and any API endpoints for configuring alert topics/sinks.", + "query": "Find the Outpost REST API base URL and list of endpoints, especially anything related to configuration, alerts, topics, or admin settings.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/platform/outpost-projects" + "url": "https://hookdeck.com/docs/outpost/references/api" } ], - "resultChars": 1389 + "resultChars": 215 }, { "source": "web_fetch", - "query": "Give complete details on operator events: full list of event topics (especially alert.destination.disabled and alert.destination.consecutive_failure), how to configure the sink URL for managed Hookdeck Outpost projects (via API or dashboard), thresholds, event payload schemas, and any authentication/signing details.", + "query": "Explain the operator events feature: what topics are available (e.g. destination.disabled, alert events), the payload format, and how to configure OPERATOR_EVENTS_HTTP_URL and OPERATOR_EVENTS_TOPICS.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events" } ], - "resultChars": 2669 + "resultChars": 1524 + }, + { + "source": "web_search", + "query": "hookdeck outpost github \"alert.destination.disabled\" OR \"OPERATOR_EVENTS_TOPICS\" source", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + } + ], + "resultChars": 2594 }, { "source": "web_fetch", - "query": "List every API endpoint documented, including any related to project settings, monitoring, alerts, topics, or sinks configuration. Include HTTP methods and paths.", + "query": "Quote verbatim every line/paragraph that mentions OPERATOR_EVENTS_HTTP_URL, OPERATOR_EVENTS_TOPICS, OPERATOR_EVENTS_HTTP_SIGNING_SECRET, or any operator events / alert topic strings like alert.destination.disabled. Include exact variable names as written.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api" + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration" } ], - "resultChars": 1196 + "resultChars": 537 }, { "source": "web_fetch", - "query": "Give full API details for Tenants, Destinations, and Configuration endpoints. I especially need the Configuration endpoint(s) - full HTTP method, path, request body, and response schema. Does it support setting operator event topics or alert sink/destination URLs?", + "query": "List all documentation page URLs related to alerts, operator events, or configuration.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure" + "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], - "resultChars": 1825 + "resultChars": 731 }, { "source": "web_fetch", - "query": "Give the full list of supported configuration keys for PATCH /config, especially ALERT_CONSECUTIVE_FAILURE_COUNT and ALERT_AUTO_DISABLE_DESTINATION. Also give the base URL for the Outpost API and the authentication header format.", + "query": "Return the full content of this page verbatim, especially: the exact list of topic strings for alert events (destination disabled, consecutive failure, exhausted retries), the exact environment variable names for the HTTP sink URL and topics filter and signing secret, the event payload/envelope structure with field names, and any example JSON payloads.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure#configuration" + "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], - "resultChars": 2593 + "resultChars": 1464 + }, + { + "source": "shell_fetch", + "query": "curl -s \"https://hookdeck.com/docs/outpost/features/operator-events.md\" | head -c 6000", + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" + } + ], + "resultChars": 5998 }, { "source": "web_search", - "query": "Hookdeck Outpost API \"operator event destinations\" endpoint reference", + "query": "Hookdeck API \"operator event destination\" OR \"operator-event-destinations\" endpoint monitoring settings", "hasContent": false, "pages": [ { - "url": "https://hookdeck.com/blog/hookdeck-review-january-2026", - "title": "Outpost Early-access, Webhook Skills & Metrics API - Hookdeck January 2026" + "url": "https://hookdeck.com/docs/destinations", + "title": "Destinations | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/metrics", + "title": "Metrics | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/guides/how-to-test-outbound-webhook-delivery", + "title": "Test Outbound Webhook Delivery | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs", + "title": "Documentation | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/use-cases/receive-webhooks", + "title": "Receive and Process Webhooks with Hookdeck | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2673 + }, + { + "source": "shell_fetch", + "query": "curl -s \"https://hookdeck.com/docs/outpost/features/operator-events.md\" | sed -n '1,40p'\necho \"----full-search-config-endpoint----\"\ncurl -s \"https://hookdeck.com/docs/api\" -o /tmp/hd_api.html -w \"%{http_code}\\n\"", + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" + } + ], + "resultChars": 2240 + }, + { + "source": "web_search", + "query": "\"dashboard.hookdeck.com/settings/project/monitoring\" outpost operator events destination configure", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/platform/outpost-projects", + "title": "Outpost Projects | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", + "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/opentelemetry", + "title": "OpenTelemetry — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/platform/event-gateway-projects", + "title": "Event Gateway Projects | Hookdeck Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" }, { "url": "https://hookdeck.com/docs/outpost/concepts", "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2176 + } + ] + }, + "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", + "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-003-operator-events.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "benchmark-outpost-004-queue-destination", + "ranAt": "2026-08-25T09:48:19.544Z", + "stage": "build", + "product": [ + "outpost" + ], + "topic": [ + "capabilities" + ], + "suite": "benchmark", + "gatedBy": "mixed", + "passed": true, + "checks": [ + { + "name": "their orders are delivered to the queue they gave us", + "passed": true + }, + { + "name": "the queue is subscribed to their order events", + "passed": true + }, + { + "name": "the queue destination carries credentials", + "passed": true + }, + { + "name": "their orders no longer go to the old endpoint", + "passed": true + }, + { + "name": "the rest of their delivery is unchanged", + "passed": true + }, + { + "name": "the other customer was left alone", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "Hookdeck Outpost API documentation base URL", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" }, { "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" }, + { + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/outpost", + "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" + } + ], + "resultChars": 1763 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost API AWS SQS destination configuration", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs", + "title": "AWS SQS — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", + "title": "Configuration Reference — Outpost Docs | Hookdeck Event Gateway Docs" + }, { "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" }, { - "url": "https://hookdeck.com/docs/outpost/publishing/events", - "title": "Publishing Events | Outpost Docs" + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/docs/outpost/destinations/webhook", - "title": "Webhook | Outpost Docs" + "url": "https://hookdeck.com/outpost", + "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" }, + { + "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", + "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2750 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost REST API authentication bearer token", + "hasContent": false, + "pages": [ { "url": "https://hookdeck.com/docs/api", "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/authentication", + "title": "Authentication & Verification | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-go-guide", + "title": "Send Webhooks with Go" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-gin-guide", + "title": "Send Webhooks with Gin" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-fastapi-guide", + "title": "Send Webhooks with FastAPI" + }, + { + "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-python-guide", + "title": "Send Webhooks with Python" + }, + { + "url": "https://hookdeck.com/docs/platform/outpost-projects", + "title": "Outpost Projects | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + } + ], + "resultChars": 2295 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost API tenants destinations topics event types", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + }, + { + "url": "https://hookdeck.com/docs/outpost/publishing/events", + "title": "Publishing Events | Outpost Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-january-2026", + "title": "Outpost Early-access, Webhook Skills & Metrics API - Hookdeck January 2026" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" + } + ], + "resultChars": 2830 + }, + { + "source": "web_search", + "query": "Hookdeck CLI outpost commands", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/cli", + "title": "The Hookdeck CLI | Hookdeck Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-cli-v2", + "title": "Hookdeck CLI v2: AI agents can query, inspect, and debug webhooks via MCP" + }, + { + "url": "https://hookdeck.com/docs/outpost/self-hosting/quickstarts/kubernetes", + "title": "Deploy on Kubernetes — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-october-2025", + "title": "CLI v1.1.0 Interactive Mode, Webhook Radar & Outpost Scaling - Hookdeck October 2025" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-march-2026", + "title": "CLI v2 with MCP, Request issues, Discord notifications - Hookdeck March 2026" + }, + { + "url": "https://hookdeck.com/docs/mcp", + "title": "MCP & Skills | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-january-2026", + "title": "Outpost Early-access, Webhook Skills & Metrics API - Hookdeck January 2026" + }, + { + "url": "https://hookdeck.com/blog/hookdeck-review-november-2025", + "title": "New Relic Metrics, Quick Filters, CLI v1.3.0 & Outpost Standard Webhooks - Hookdeck November 2025" } ], - "resultChars": 2657 + "resultChars": 2503 }, { "source": "web_fetch", - "query": "List all documentation page URLs related to operator events, alerts, monitoring, or destinations configuration API.", + "query": "What are the exact configuration fields required for AWS SQS destination? Include field names, required vs optional, and example request body.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/llms.txt" + "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs" } ], - "resultChars": 1320 + "resultChars": 867 }, { "source": "web_fetch", - "query": "I need the exact API steps to register an operator event destination for a managed Hookdeck Outpost project via the REST API (not the dashboard). Specifically: is there a reserved/special tenant ID used for operator event destinations? What is the exact HTTP request (method, path, body) to create a webhook destination that receives operator events like alert.destination.disabled? Quote any code samples verbatim.", + "query": "How do topics work? How are destinations filtered by topics? Can destinations subscribe to specific topics or just wildcard all?", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" + "url": "https://hookdeck.com/docs/outpost/api/topics" + } + ], + "resultChars": 215 + }, + { + "source": "web_fetch", + "query": "What is the base URL for the Outpost API and authentication method?", + "hasContent": true, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/overview" + } + ], + "resultChars": 592 + }, + { + "source": "web_search", + "query": "Hookdeck Outpost API destinations create endpoint documentation", + "hasContent": false, + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/api", + "title": "REST API Reference | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", + "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + }, + { + "url": "https://hookdeck.com/docs/outpost/overview", + "title": "Overview — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/create-a-destination", + "title": "Destinations | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" + }, + { + "url": "https://hookdeck.com/docs/use-cases/send-webhooks", + "title": "Send Webhooks With Hookdeck | Hookdeck Event Gateway Docs" } ], - "resultChars": 917 + "resultChars": 2921 }, { "source": "web_search", - "query": "\"operator-events/destinations\" outpost hookdeck", + "query": "site:outpost.hookdeck.com/docs/api destinations", "hasContent": false, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", - "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/docs/outpost/destinations/webhook", + "title": "Webhook | Outpost Docs" + }, + { + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", + "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" }, { "url": "https://hookdeck.com/docs/platform/outpost-projects", "title": "Outpost Projects | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", - "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" + "url": "https://hookdeck.com/docs/outpost/concepts", + "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" }, { - "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", - "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" + "url": "https://hookdeck.com/blog/outpost-open-source-webhooks-event-destinations-infrastructure", + "title": "Introducing Outpost: Open Source Outbound Webhooks and Event Destinations Infrastructure" }, { "url": "https://hookdeck.com/outpost", "title": "Outpost - Managed & Self-hosted infrastructure for sending webhooks by Hookdeck" - }, - { - "url": "https://hookdeck.com/docs/outpost/publishing/events", - "title": "Publishing Events | Outpost Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs", - "title": "AWS SQS — Outpost Docs | Hookdeck Event Gateway Docs" } ], - "resultChars": 2578 + "resultChars": 2244 }, { "source": "web_fetch", - "query": "Give the exact JSON request body example for POST /tenants/{tenant_id}/destinations to create a webhook destination, including all field names (type, topics, config object with url, etc.)", + "query": "What is the REST API endpoint to create a destination? Show the full POST request URL, body structure, and how topics are filtered.", "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/destinations/webhook" + "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui" } ], - "resultChars": 743 - } - ] - }, - "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", - "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-003-operator-events.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:27:15.505Z", - "stage": "build", - "product": [ - "outpost" - ], - "topic": [ - "capabilities" - ], - "suite": "benchmark", - "gatedBy": "mixed", - "passed": true, - "checks": [ - { - "name": "their orders are delivered to the queue they gave us", - "passed": true - }, - { - "name": "the queue is subscribed to their order events", - "passed": true - }, - { - "name": "the queue destination carries credentials", - "passed": true - }, - { - "name": "their orders no longer go to the old endpoint", - "passed": true - }, - { - "name": "the rest of their delivery is unchanged", - "passed": true - }, - { - "name": "the other customer was left alone", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ + "resultChars": 1370 + }, { - "source": "web_search", - "query": "Hookdeck Outpost API tenants destinations reference", - "hasContent": false, + "source": "web_fetch", + "query": "What is the base URL and the exact REST endpoint for creating a destination? Show the full path and method.", + "hasContent": true, "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", - "title": "Tenant Portal — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", - "title": "Migrate Existing Webhooks to Outpost — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", - "title": "Building Your Own UI — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/concepts", - "title": "Concepts — Outpost Docs | Hookdeck Event Gateway Docs" - }, - { - "url": "https://hookdeck.com/docs/outpost/destinations/webhook", - "title": "Webhook | Outpost Docs" + "url": "https://hookdeck.com/docs/outpost/api" } ], - "resultChars": 2512 + "resultChars": 771 } ] }, @@ -7385,7 +7653,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T16:04:27.436Z", + "ranAt": "2026-08-25T10:19:40.439Z", "stage": "build", "product": [ "outpost" @@ -7421,291 +7689,151 @@ "outpost" ], "loaded": [ - "hookdeck", "outpost" ] }, "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/llms.txt | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 11412 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api" - } - ], - "resultChars": 613299 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api.md" - } - ], - "resultChars": 5554 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/features/operator-events.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" - } - ], - "resultChars": 8210 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/resources.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/resources.md" - } - ], - "resultChars": 5733 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost disabled destination alert consecutive delivery failures threshold Outpost", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '1,260p'\"", + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/features/operator-events", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" + "url": "https://hookdeck.com/docs/outpost/features/operator-events" } - ], - "resultChars": 5832 + ] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/inspect.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/inspect.md" - } - ], - "resultChars": 7595 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost Config API monitoring settings operator events HTTP url topic update", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/outpost-projects.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/outpost-projects.md" - } - ], - "resultChars": 5841 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api config API operation events http url topics", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/organizations.md | sed -n '1,220p'\"", + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { - "url": "https://hookdeck.com/docs/platform/organizations.md" + "url": "https://hookdeck.com/docs/outpost/api" } - ], - "resultChars": 3442 + ] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/llms.txt | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/llms.txt" - } - ], - "resultChars": 22682 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Monitoring settings\" \"Config API\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/llms.txt | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/llms.txt" - } - ], - "resultChars": 9141 + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"operation events\" \"api\" \"settings\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/api.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/api.md" - } - ], - "resultChars": 8525 + "source": "web_search", + "query": "site:hookdeck.com/docs \"Config API\" \"project settings\" hookdeck monitoring", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/platform/project-settings.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/platform/project-settings.md" - } - ], - "resultChars": 27518 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform/outpost-projects API project settings monitoring Config API", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/api/configure.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/api/configure.md" - } - ], - "resultChars": 7829 + "source": "web_search", + "query": "site:github.com/hookdeck/outpost openapi yaml config operation events http url", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/issues.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/issues.md" - } - ], - "resultChars": 10152 + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"getManagedConfig\" \"updateManagedConfig\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/issue-triggers.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/issue-triggers.md" - } - ], - "resultChars": 9240 + "source": "web_search", + "query": "site:raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml OPERATION_EVENTS_HTTP_URL", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/llms.txt | rg -n 'operator|monitor|alert|destination|config'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 4305 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform OPERATION_EVENTS_HTTP_URL", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api.md | rg -n 'operator-events|ALERT_CONSECUTIVE_FAILURE_COUNT|ALERT_AUTO_DISABLE_DESTINATION|monitoring|config'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api.md" - } - ], - "resultChars": 246 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform/project settings monitoring api operator events hookdeck api", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/features/operator-events.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" - } - ], - "resultChars": 8210 + "source": "web_search", + "query": "site:hookdeck.com/docs platform project settings API monitoring outpost operator events sink", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '1,260p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 5832 + "source": "web_search", + "query": "site:hookdeck.com/docs/api projects settings monitoring alert webhook endpoint hookdeck", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api.md | rg -n 'Configuration|/config|operator-events|monitoring|destination-types'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api.md" - } - ], - "resultChars": 246 + "source": "web_search", + "query": "site:hookdeck.com/docs/api projects organization settings alerting outpost operator events", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | rg -n 'Configuration|/config|monitoring|operator|alert|destination|topics'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 6126 + "source": "web_search", + "query": "site:raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml \"operator event destinations\"", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/features/operator-events.md | rg -n 'Config API|Monitoring settings|ALERT_AUTO_DISABLE_DESTINATION|ALERT_CONSECUTIVE_FAILURE_COUNT|sink|topics'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" - } - ], - "resultChars": 3408 + "source": "web_search", + "query": "site:github.com/hookdeck operator event destinations webhook url project settings API", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '836,980p'\"", + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/destinations/webhook", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" + "url": "https://hookdeck.com/docs/outpost/destinations/webhook" } - ], - "resultChars": 6503 + ] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/resources.md | rg -n 'operator|monitor|config|sink|destination|topic'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/resources.md" - } - ], - "resultChars": 2101 + "source": "web_search", + "query": "site:hookdeck.com/docs/platform/projects api project settings hookdeck", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/llms.txt | rg -n 'operator-events|destination-types|monitoring|destination'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/llms.txt" - } - ], - "resultChars": 3400 + "source": "web_search", + "query": "site:hookdeck.com/docs notifications webhook specific events api hookdeck project settings", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | sed -n '322,440p'\"", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api/configure.md" - } - ], - "resultChars": 2569 + "source": "web_search", + "query": "site:hookdeck.com/docs \"PUT /2025-07-01\" \"operator events\" hookdeck", + "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md | sed -n '1,220p'\"", + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"operator-events/destinations\"", + "pages": [] + }, + { + "source": "web_search", + "query": "https://hookdeck.com/docs/outpost/self-hosting/changelog/upgrade-v0.16", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md" + "url": "https://hookdeck.com/docs/outpost/self-hosting/changelog/upgrade-v0.16" } - ], - "resultChars": 6070 + ] } ] }, @@ -7724,7 +7852,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:34:20.878Z", + "ranAt": "2026-08-25T10:22:06.313Z", "stage": "build", "product": [ "outpost" @@ -7773,47 +7901,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost update destination tenant_id destinations PUT Hookdeck Outpost", - "pages": [] - }, - { - "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/api", - "pages": [ - { - "url": "https://hookdeck.com/docs/outpost/api" - } - ] - }, - { - "source": "web_search", - "query": "'Update Destination' in https://hookdeck.com/docs/outpost/api", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api destination update outpost destination_id", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api \"Create or Update Destination\" Hookdeck Outpost", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api \"Update Destination\" \"Outpost API Reference\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api \"PATCH /api/v1/tenants//destinations/\"", - "pages": [] - } - ] + "calls": [] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", @@ -11951,7 +12039,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T15:59:29.050Z", + "ranAt": "2026-08-25T10:13:38.069Z", "stage": "build", "product": [ "outpost" @@ -11988,31 +12076,50 @@ "calls": [ { "source": "web_search", - "query": "Hookdeck API alerts destination disabled delivery failures threshold endpoint project API", + "query": "site:docs.hookdeck.com Hookdeck API destinations alerts disable destination consecutive delivery failures operator alerts", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs notification settings API Hookdeck webhooks notification channels", + "query": "Hookdeck alerts destination failure operator alert webhook", "pages": [] }, { "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/features/operator-events", + "query": "https://hookdeck.com/docs/issues", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/operator-events" + "url": "https://hookdeck.com/docs/issues" + } + ] + }, + { + "source": "web_search", + "query": "'notifications/webhooks' in https://hookdeck.com/docs/issues", + "pages": [] + }, + { + "source": "web_search", + "query": "https://hookdeck.com/docs/issues/issue-triggers", + "pages": [ + { + "url": "https://hookdeck.com/docs/issues/issue-triggers" } ] }, { "source": "web_search", - "query": "'Configuration API' in https://hookdeck.com/docs/outpost/features/operator-events", + "query": "site:hookdeck.com/docs issue triggers Hookdeck delivery attempts threshold", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/api issue triggers Hookdeck API", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api config api outpost operator events settings", + "query": "'notifications' in https://hookdeck.com/docs/api", "pages": [] }, { @@ -12026,37 +12133,56 @@ }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"notification\" \"project\" Hookdeck API Reference", + "query": "site:hookdeck.com/docs/api Hookdeck authentication Bearer API key", "pages": [] }, { "source": "web_search", - "query": "apis.io Hookdeck Notifications API OpenAPI", + "query": "https://hookdeck.com/docs/api", + "pages": [ + { + "url": "https://hookdeck.com/docs/api" + } + ] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/api \"notifications/webhooks\" \"api.hookdeck.com\"", "pages": [] }, { "source": "web_search", - "query": "'notification' in https://github.com/hookdeck/outpost/blob/main/docs/apis/openapi.yaml", + "query": "site:hookdeck.com/docs/api \"GET /2025-07-01\" Hookdeck project", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/outpost operator event destinations API", + "query": "site:hookdeck.com/docs/api Bearer Token Authentication Hookdeck API key", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/outpost \"operator event destinations\" \"Config API\"", + "query": "site:hookdeck.com/docs/api \"Base URL\" Hookdeck REST API", "pages": [] }, { "source": "web_search", - "query": "\"operator event destinations\" hookdeck github", + "query": "site:hookdeck.com/docs/api \"2025-07-01/notifications/webhooks\"", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/outpost/api openapi json operator event destinations", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"operator events\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api reference openapi outpost", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api openapi api.outpost.hookdeck.com", "pages": [] }, { @@ -12070,22 +12196,87 @@ }, { "source": "web_search", - "query": "site:github.com/hookdeck/outpost \"OPERATOR_EVENTS_HTTP_URL\"", + "query": "site:hookdeck.com/docs/outpost/api operator events config api key topics sink", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"OPERATION_EVENTS_HTTP_URL\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" \"curl\" \"settings\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost \"Config API\" endpoint \"PUT\" \"config\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api config api endpoint operator events", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api monitoring operator events outpost api", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost operator event destinations API", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/outpost/api \"OPERATION_EVENTS_HTTP_URL\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost OPERATION_EVENTS_HTTP_URL", + "pages": [] + }, + { + "source": "web_search", + "query": "'OPERATOR_EVENTS_HTTP_URL' in https://github.com/hookdeck/outpost/blob/main/docs/apis/openapi.yaml", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"OPERATION_EVENTS_HTTP_URL\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"operation_events\" \"api/v1\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/hookdeck/outpost \"settings/project/monitoring\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:hookdeck.com/docs/platform project settings monitoring API hookdeck", "pages": [] }, { "source": "web_search", - "query": "Hookdeck Notifications API webhook channel create endpoint", + "query": "site:npmjs.org hookdeck cli package monitoring outpost", "pages": [] }, { "source": "web_search", - "query": "site:github.com/hookdeck/outpost \"operator event destinations\" \"api\"", + "query": "\"operator event destinations\" Hookdeck", "pages": [] }, { "source": "web_search", - "query": "site:github.com/hookdeck/outpost operator_events hookdeck outpost code", + "query": "https://dashboard.hookdeck.com/settings/project/monitoring", "pages": [] } ] @@ -12105,7 +12296,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:32:01.863Z", + "ranAt": "2026-08-25T10:14:57.662Z", "stage": "build", "product": [ "outpost" @@ -12150,17 +12341,12 @@ "calls": [ { "source": "web_search", - "query": "site:hookdeck.com docs Hookdeck API authentication X-API-KEY Bearer", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/aws sqs hookdeck destination API create destination request body", + "query": "Hookdeck API connections destinations sources documentation orders queue SQS official", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api create destination aws_sqs event gateway Hookdeck API", + "query": "site:hookdeck.com/docs hookdeck api connections destinations REST create update official", "pages": [] }, { @@ -12174,30 +12360,25 @@ }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"Create a destination\" Hookdeck Event Gateway", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/api Hookdeck API Bearer token base url /2025-07-01", + "query": "site:hookdeck.com/docs/api destinations connections openapi Hookdeck REST API", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"List connections\" Hookdeck API", + "query": "site:hookdeck.com/docs/outpost/api openapi json hookdeck outpost", "pages": [] }, { "source": "web_search", - "query": "site:hookdeck.com/docs/api \"api.hookdeck.com\" Hookdeck", + "query": "site:hookdeck.com/docs/outpost/api/2025-07-01 outpost api reference hookdeck", "pages": [] }, { "source": "web_search", - "query": "https://hookdeck.com/docs/api", + "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { - "url": "https://hookdeck.com/docs/api" + "url": "https://hookdeck.com/docs/outpost/api" } ] } @@ -14027,7 +14208,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T16:11:00.373Z", + "ranAt": "2026-08-25T10:30:12.675Z", "stage": "build", "product": [ "outpost" @@ -14069,37 +14250,34 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "site:hookdeck.com/docs/outpost notifications destination disabled consecutive failures Outpost", - "pages": [] - }, - { - "source": "web_search", - "query": "\"managed by your operator event destinations\" Hookdeck", - "pages": [] - }, - { - "source": "web_search", - "query": "site:hookdeck.com/docs/api monitoring notification destination Hookdeck project monitoring API", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"cat .agents/skills/outpost/references/outpost-verify.md; printf '\\\\n--- DOC INDEX MATCHES ---\\\\n'; curl -fsSL https://hookdeck.com/docs/outpost/llms.txt | rg -i 'alert|notification|disable|failure|attempt|project|setting' | head -100; printf '\\\\n--- OPENAPI MATCHES ---\\\\n'; curl -fsSL https://raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml -o /tmp/outpost-openapi.yaml; rg -n -i 'alert|notification|disable|failure|attempt|project|setting' /tmp/outpost-openapi.yaml | head -200\"", + "pages": [ + { + "url": "https://hookdeck.com/docs/outpost/llms.txt" + } + ], + "resultChars": 14598 }, { - "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/features/operator-events.md", + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- OPERATOR EVENTS DOC ---'; curl -fsSL https://hookdeck.com/docs/outpost/features/operator-events.md; printf '%s\\\\n' '--- CONFIG OPENAPI ---'; sed -n '3010,3185p' /tmp/outpost-openapi.yaml; sed -n '3240,3290p' /tmp/outpost-openapi.yaml; rg -n '\"'^ /.*config|operationId:.*[Cc]onfig|Configuration'\"' /tmp/outpost-openapi.yaml | tail -50\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } - ] + ], + "resultChars": 18777 }, { - "source": "web_search", - "query": "https://hookdeck.com/docs/outpost/features/operator-events", + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '2960,3175p' /tmp/outpost-openapi.yaml; sed -n '3380,3475p' /tmp/outpost-openapi.yaml; printf '\\\\n--- CURL QUICKSTART AUTH/BASE ---\\\\n'; curl -fsSL https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md | sed -n '1,180p'\"", "pages": [ { - "url": "https://hookdeck.com/docs/outpost/features/operator-events" + "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md" } - ] + ], + "resultChars": 6070 } ] }, @@ -14118,7 +14296,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:38:31.684Z", + "ranAt": "2026-08-25T10:32:34.410Z", "stage": "build", "product": [ "outpost" @@ -15899,7 +16077,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", - "ranAt": "2026-08-24T16:07:45.281Z", + "ranAt": "2026-08-25T10:25:42.942Z", "stage": "build", "product": [ "outpost" @@ -15936,18 +16114,23 @@ "calls": [ { "source": "web_search", - "query": "site:hookdeck.com/docs outpost destination disabled webhook attempts alert", + "query": "site:hookdeck.com/docs Outpost destination disabled alert notification delivery failures", "pages": [] }, { "source": "web_search", - "query": "Hookdeck Outpost API reference", + "query": "site:outpost.hookdeck.com/docs \"disabled\" destination", "pages": [] }, { - "source": "web_search", - "query": "site:hookdeck.com/docs/api monitoring operator events config API Hookdeck", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"for path in \\\\\n /docs/outpost/overview \\\\\n /docs/outpost/features \\\\\n /docs/outpost/features/delivery-attempts \\\\\n /docs/outpost/features/automatic-disable \\\\\n /docs/outpost/features/notifications \\\\\n /docs/outpost/api \\\\\n /docs/api \\\\\n /docs/outpost/quickstarts/overview; do \\\\\n url=\\\"https://hookdeck.com\"'$path\"; code=$(curl -sS -L -o \"/tmp/doc$(echo \"$path\"|tr / _)\" -w '\"'%{http_code}' \\\"\"'$url\"); printf '\"'%s %s %s\\\\n' \\\"\"'$code\" \"$(wc -c <\"/tmp/doc$(echo \"$path\"|tr / _)\")\" \"$path\"; done; \nprintf '\"'%s\\\\n' '--- site map outpost paths ---'; curl -sS -L https://hookdeck.com/sitemap-index.xml | head -c 1000; echo; curl -sS -L https://hookdeck.com/sitemap-0.xml -o /tmp/sitemap || true; rg -o 'https://hookdeck.com/[\"'^<]+'\"' /tmp/sitemap | rg 'outpost|issue|notification|alert' | head -200\"", + "pages": [ + { + "url": "https://hookdeck.com/sitemap-index.xml" + } + ], + "resultChars": 9134 }, { "source": "web_search", @@ -15975,7 +16158,7 @@ "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", - "ranAt": "2026-08-24T14:36:10.449Z", + "ranAt": "2026-08-25T10:28:01.601Z", "stage": "build", "product": [ "outpost" @@ -16020,12 +16203,7 @@ "calls": [ { "source": "web_search", - "query": "site:hookdeck.com/docs Outpost API subdomain destinations SQS Hookdeck", - "pages": [] - }, - { - "source": "web_search", - "query": "Hookdeck Outpost API GitHub destinations subscriptions API", + "query": "site:hookdeck.com/docs/api destinations create AWS SQS Hookdeck API", "pages": [] } ] diff --git a/results/runs/2026-08-25T10-33-18-000Z.json b/results/runs/2026-08-25T10-33-18-000Z.json new file mode 100644 index 0000000..5bd768d --- /dev/null +++ b/results/runs/2026-08-25T10-33-18-000Z.json @@ -0,0 +1,16578 @@ +{ + "schemaVersion": 1, + "publishedAt": "2026-08-25T10:33:18.000Z", + "runId": "local-2026-08-25-sweep", + "counts": { + "rows": 114, + "experiments": 6, + "evals": 19, + "passed": 103, + "runs": 1 + }, + "results": [ + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "benchmark-alerting-001-delivery-alerts", + "ranAt": "2026-08-19T14:18:38.238Z", + "stage": "build", + "product": [ + "event-gateway" + ], + "topic": [ + "alerting" + ], + "suite": "benchmark", + "skills": { + "available": [ + "hookdeck", + "event-gateway" + ], + "loaded": [ + "event-gateway" + ] + }, + "passed": true, + "checks": [ + { + "name": "an alert exists for delivery failures", + "passed": true + }, + { + "name": "the alert is enabled", + "passed": true + }, + { + "name": "the alert covers the fulfilment connection", + "passed": true + }, + { + "name": "the alert has a notification channel configured", + "passed": true + } + ], + "attempts": 1, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://hookdeck.com/docs/api/issue-triggers.md", + "pages": [ + { + "url": "https://hookdeck.com/docs/api/issue-triggers.md" + } + ], + "resultChars": 66518 + }, + { + "source": "shell_fetch", + "query": "curl -s https://hookdeck.com/docs/issue-triggers.md 2>&1 | head -300", + "pages": [ + { + "url": "https://hookdeck.com/docs/issue-triggers.md" + } + ], + "resultChars": 9388 + }, + { + "source": "shell_fetch", + "query": "echo \"=== existing issue triggers ===\"; curl -s https://api.hookdeck.com/2025-07-01/issue-triggers -H \"Authorization: Bearer $HOOKDECK_API_KEY\" | head -c 3000\necho\necho \"=== notifications docs ===\"\ncurl -s https://hookdeck.com/docs/api/notifications.md 2>&1 | head -c 100\necho\necho \"=== issues doc channels section ===\"\ncurl -s https://hookdeck.com/docs/issues.md 2>&1 | grep -n -i -A5 \"channel\"", + "pages": [ + { + "url": "https://hookdeck.com/docs/api/notifications.md" + }, + { + "url": "https://hookdeck.com/docs/issues.md" + } + ], + "resultChars": 2272 + }, + { + "source": "shell_fetch", + "query": "curl -s https://api.hookdeck.com/2025-07-01/issue-triggers -H \"Authorization: Bearer $HOOKDECK_API_KEY\"\necho\necho \"=== project notification settings ===\"\ncurl -s https://hookdeck.com/docs/issues.md 2>&1 | sed -n '1,40p'", + "pages": [ + { + "url": "https://hookdeck.com/docs/issues.md" + } + ], + "resultChars": 3193 + } + ] + }, + "toolCalls": [ + { + "endpoint": "Skill", + "body": { + "skill": "event-gateway" + }, + "name": "unknown", + "loadedSkills": [ + "event-gateway" + ], + "result": "Launching skill: event-gateway", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "hookdeck whoami", + "description": "Show current Hookdeck org/project context" + }, + "name": "shell", + "command": "hookdeck whoami", + "error": "Exit code 1\nYou aren't authenticated yet.\n\nNo terminal is attached, so browser sign-in cannot run.\n\nAuthenticate without a terminal using one of:\n hookdeck ci --api-key (or set HOOKDECK_API_KEY)\n hookdeck login --cli-key \n\nOr run `hookdeck login` in an interactive terminal.", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "echo \"HOOKDECK_API_KEY set: ${HOOKDECK_API_KEY:+yes}\"; hookdeck ci --api-key \"$HOOKDECK_API_KEY\" 2>&1; hookdeck whoami 2>&1", + "description": "Authenticate CLI with API key and show identity" + }, + "name": "shell", + "command": "echo \"HOOKDECK_API_KEY set: ${HOOKDECK_API_KEY:+yes}\"; hookdeck ci --api-key \"$HOOKDECK_API_KEY\" 2>&1; hookdeck whoami 2>&1", + "result": "HOOKDECK_API_KEY set: yes\ntime=\"Wed, 19 Aug 2026 14:15:41 UTC\" level=info msg=\"The Hookdeck CLI is configured on project evals-ci in organization Automated Testing\n\"\n\nUsing profile default (use -p flag to use a different config profile)\n\nLogged in as 8e127ae8abb5 () on project evals-ci in organization Automated Testing\nProject type: Gateway", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "hookdeck gateway connection list 2>&1", + "description": "List connections in project" + }, + "name": "shell", + "command": "hookdeck gateway connection list 2>&1", + "result": "Found 1 connection(s):\n\norders-to-fulfilment\n ID: web_BFL5GHb0FGi4\n Source: orders (src_8daq3ap8g6nc7g) [WEBHOOK]\n Destination: fulfilment-api (des_ilE1tIK8Koli) [HTTP]\n Status: active", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "cat /tmp/sandbox-9ec775af/.claude/skills/event-gateway/references/monitoring-debugging.md 2>&1 | head -200", + "description": "Read monitoring/debugging reference" + }, + "name": "shell", + "command": "cat /tmp/sandbox-9ec775af/.claude/skills/event-gateway/references/monitoring-debugging.md 2>&1 | head -200", + "result": "# Monitoring & Debugging\n\n## Contents\n\n- [Before querying or running metrics](#before-querying-or-running-metrics)\n- [Monitoring](#monitoring)\n - [Event Lifecycle](#event-lifecycle)\n - [Data Model](#data-model)\n - [Event Statuses](#event-statuses)\n - [Where to look](#where-to-look)\n- [Debugging](#debugging)\n - [Troubleshooting Flowchart](#troubleshooting-flowchart)\n - [Issues and Notifications](#issues-and-notifications)\n - [Replay](#replay)\n- [CLI: list and inspect](#cli-querying)\n- [CLI: metrics](#cli-metrics)\n- [Documentation](#documentation)\n\nHow to monitor webhook deliveries, debug failures, and replay events. For monitoring and debugging, the **CLI** is usually the right first touch (TUI, list/get, metrics, retry). The **API** is valid for querying and for automation driven from application or test code. See **[SKILL.md — Development & Operations](../SKILL.md#development--operations)** for full CLI vs API vs Dashboard guidance. Querying (events, requests, attempts, metrics) can also be done via the API — see the [API inspect docs](https://hookdeck.com/docs/api/inspect.md).\n\n**Prerequisite:** Before any `hookdeck …` commands in this file, [Hookdeck CLI installed](cli-workflows.md#hookdeck-cli-prerequisite).\n\n## Before querying or running metrics {#before-querying-or-running-metrics}\n\nAll list, inspect, and metrics commands are scoped to the **current [organization and project](https://hookdeck.com/docs/projects)** (organization = top-level account; project = where your sources, connections, and destinations live).\n\n- [ ] Run `hookdeck whoami` and **show the user the output**\n- [ ] **Unless** the user has very clearly identified org/project and whoami is an exact match, ask them to confirm before running queries or metrics\n- [ ] If not correct, use `hookdeck project list` to see options, then `hookdeck project use `\n- [ ] Run `hookdeck whoami` again, show the output, and (unless clear match) ask the user to confirm again before running queries or metrics\n\n**Common mistake:** Running a metrics query without verifying org/project — data can come from the wrong organization (e.g. \"Automated Testing\" instead of \"prod\"). **Correct approach:** Run `hookdeck whoami` and show the output. If the user hasn't clearly specified org/project or it doesn't match, ask them to confirm; if not correct, switch and confirm again. Only then run the query.\n\n## Monitoring {#monitoring}\n\nEvent lifecycle, data model, event statuses, and where to look (CLI TUI, Web Console, Dashboard).\n\n### Event Lifecycle\n\n1. Webhook provider sends a request to your Source URL (`https://hkdk.events/xxx`)\n2. Hookdeck creates a [Request](https://hookdeck.com/docs/requests) (the raw inbound data)\n3. Each Connection on that Source creates an [Event](https://hookdeck.com/docs/events) (processed, routed delivery)\n4. Hookdeck delivers the Event to the Destination, creating an [Attempt](https://hookdeck.com/docs/events)\n5. If delivery fails (non-2xx), Hookdeck retries according to the Connection's retry rules\n\n### Data Model\n\n| Entity | What it is |\n|--------|------------|\n| [Request](https://hookdeck.com/docs/requests) | Raw inbound webhook received by a Source |\n| [Event](https://hookdeck.com/docs/events) | Processed delivery routed through a Connection |\n| [Attempt](https://hookdeck.com/docs/events) | A single delivery try to a Destination |\n\nOne Request can produce multiple Events (fan-out). Each Event has one or more Attempts.\n\n### Event Statuses\n\n| Status | Meaning |\n|--------|---------|\n| `SUCCESSFUL` | Destination returned a 2xx response |\n| `FAILED` | All retry attempts exhausted, destination still returning non-2xx |\n| `QUEUED` | Scheduled for delivery or retry |\n| `HOLD` | Paused by a delay rule or manual hold |\n\n### Where to look\n\n#### CLI TUI\n\nBuilt into `hookdeck listen`. View events in real time, inspect request/response details, replay events -- all in the terminal.\n\n#### Web Console\n\nWhen not using an Event Gateway project, the CLI provides a link to a web interface for event inspection and replay.\n\n#### Dashboard\n\nWhen using an Event Gateway project, the [Hookdeck Dashboard](https://dashboard.hookdeck.com) provides:\n\n- **Events view**: Inspect Event payloads, headers, response, and Attempts\n- **Requests view**: See raw inbound webhooks before routing\n- **Issues**: Track patterns of failures (delivery issues, transformation errors, backpressure)\n- **Replay**: Re-deliver individual Events or bulk replay filtered sets\n- **Bookmarks**: Save representative Requests for repeated testing\n\n## Debugging {#debugging}\n\nTroubleshooting flowchart, issues and notifications, and replay.\n\n### Troubleshooting Flowchart\n\n**Events not arriving?**\n\n1. Is the provider sending to the correct Source URL?\n2. Is `hookdeck listen` running?\n3. If Source Authentication is configured, are the credentials correct? Failed auth returns `403` and no Event is created.\n\n**Events arriving but delivery failing?**\n\n1. Check the Attempt response in the CLI TUI or web UI\n2. Is your handler running and listening on the correct port?\n3. Is the endpoint path correct (e.g., `/webhooks` vs `/webhook`)?\n4. Is the handler returning a 2xx status? Non-2xx triggers retries.\n\n**Signature verification failing in your handler?**\n\n1. Is `HOOKDECK_WEBHOOK_SECRET` set correctly?\n2. Are you using the raw request body (not parsed JSON)?\n3. Are you using base64 encoding (not hex)?\n4. See [verification-code.md](verification-code.md) for debugging code\n\n### Issues and Notifications\n\n[Issues](https://hookdeck.com/docs/issues) are automatically created when problems are detected:\n\n| Issue type | Trigger |\n|------------|---------|\n| Delivery | Consecutive failed delivery attempts |\n| Transformation | JavaScript errors in transform rules |\n| Backpressure | Events queuing faster than they can be delivered |\n\nIssue lifecycle: `OPENED` -> `ACKNOWLEDGED` -> `RESOLVED` or `IGNORED`\n\n[Issue triggers](https://hookdeck.com/docs/issue-triggers) control what creates issues. [Notifications](https://hookdeck.com/docs/issue-triggers) can be sent to Email, Slack, PagerDuty, Microsoft Teams, or a webhook.\n\n### Replay\n\n**Single event replay**: Use the CLI TUI, web console, or Dashboard to replay any event.\n\n**Bulk retry via API**:\n\n```sh\ncurl -X POST https://api.hookdeck.com/$API_VERSION/bulk/events/retry \\\n -H \"Authorization: Bearer $HOOKDECK_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"query\": {\"webhook_id\": \"web_xxx\", \"status\": \"FAILED\"}}'\n```\n\n**[Bookmarks](https://hookdeck.com/docs/bookmarks)**: Save a Request as a bookmark in the Dashboard, then replay it whenever you need to re-test.\n\n## CLI: list and inspect {#cli-querying}\n\nWhen to use: detailed search, inspecting or retrying individual request, event, or attempt. From the terminal you can list, inspect, and retry without opening the Dashboard. Order reflects data flow: request → events → attempts.\n\n```sh\nhookdeck gateway request list\nhookdeck gateway request get req_xxx\nhookdeck gateway request retry req_xxx\nhookdeck gateway event list\nhookdeck gateway event get evt_xxx\nhookdeck gateway event retry evt_xxx\nhookdeck gateway attempt list --event-id evt_xxx\nhookdeck gateway attempt get att_xxx\n```\n\nSee [Request commands](https://hookdeck.com/docs/cli/request.md), [Event commands](https://hookdeck.com/docs/cli/event.md), and [Attempt commands](https://hookdeck.com/docs/cli/attempt.md) for full options. For aggregated metrics (volume, failure rate, backlog), see [CLI: metrics](#cli-metrics) in this file.\n\n## CLI: metrics {#cli-metrics}\n\nWhen to use: event volume, failure rates, which connections had the most events, backlog. Questions like *which connections had the most events?*, *what's our failure rate?*, or *where is backlog building?* are answered from the terminal with the metrics subcommands below. For listing or inspecting individual requests, events, or attempts, see [CLI: list and inspect](#cli-querying). Metrics over time are also available in the [Dashboard](https://dashboard.hookdeck.com) ([Metrics page](https://dashboard.hookdeck.com/metrics) and Source/Connection/Destination pages). The CLI uses `hookdeck gateway metrics` and its subcommands. All CLI commands require a date range (`--start`, `--end`, ISO 8601) and at least one `--measures` value; optional filters include `--granularity`, `--dimensions`, `--source-id`, `--destination-id`, `--connection-id`, and `--status`. See [Metrics](https://hookdeck.com/docs/metrics) and the [CLI metrics reference](https://hookdeck.com/docs/cli/metrics.md) for full reference.\n\n| Subcommand | Purpose |\n|------------|---------|\n| `metrics events` | Event volume/failure rates plus queue depth, pending, and issue-grouped views (via measures/dimensions) |\n| `metrics requests` | Request acceptance vs rejection counts |\n| `metrics attempts` | Delivery latency and success/failure |\n| `metrics transformations` | Transformation run counts and error rate |\n\n**Example commands (use cases):**\n\nEvent volume and failure rate over time:\n\n```sh\nhookdeck gateway metrics events --start 2026-02-01T00:00:00Z --end 2026-02-25T00:00:00Z --granularity 1d --measures count,failed_count,error_rate\n```\n\nRequest acceptance vs rejection:\n\n```sh\nhookdeck gateway metrics requests --start 2026-02-01T00:00:00Z --end 2026-02-25T00:00:00Z --measures count,accepted_count,rejected_count\n```\n\nDelivery latency (attempts):\n\n```sh\nhookdeck gateway metrics attempts --start 2026-02-01T00:00:00Z --end 2026-02-25T00:00:00Z --measures response_latency_avg,response_latency_p95\n```\n\nQueue backlog (use `events` with queue measures and optional `--destination-id`):\n\n```sh\nhookdeck gateway metrics events --start 2026-02-01T00:00:00Z --end 2026-02-25T00:00:00Z --measures max_depth,max_age --destination-id des_xxx\n```\n\nPending events over time (use `events` with `pending` measure):\n\n```sh\nhookdeck gateway metrics events --start 2026-02-01T00:00:00Z --end 2026-02-25T00:00:00Z --granularity 1h --measures pending,count\n```\n\nEvents grouped by issue (use `events` with `--dimensions issue_id` and `--issue-id`):", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "grep -n -i \"issue\" /tmp/sandbox-9ec775af/.claude/skills/event-gateway/references/cli-workflows.md", + "description": "Search cli-workflows reference for issue trigger commands" + }, + "name": "shell", + "command": "grep -n -i \"issue\" /tmp/sandbox-9ec775af/.claude/skills/event-gateway/references/cli-workflows.md", + "result": "(Bash completed with no output)", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "hookdeck gateway --help 2>&1 | grep -i issue; echo \"---\"; hookdeck gateway issue-trigger --help 2>&1", + "description": "Check CLI for issue-trigger subcommands" + }, + "name": "shell", + "command": "hookdeck gateway --help 2>&1 | grep -i issue; echo \"---\"; hookdeck gateway issue-trigger --help 2>&1", + "result": " issue Manage your issues [BETA]\n---\nCommands for managing Event Gateway sources, destinations, connections,\ntransformations, events, requests, metrics, and MCP server.\n\nThe gateway command group provides full access to all Event Gateway resources.\n\nUsage:\n hookdeck gateway [command]\n\nExamples:\n # List connections\n hookdeck gateway connection list\n\n # Create a source\n hookdeck gateway source create --name my-source --type WEBHOOK\n\n # Query event metrics\n hookdeck gateway metrics events --start 2026-01-01T00:00:00Z --end 2026-02-01T00:00:00Z\n\n # Start the MCP server for AI agent access\n hookdeck gateway mcp\n\nAvailable Commands:\n attempt Inspect delivery attempts [BETA]\n connection Manage your connections [BETA]\n destination Manage your destinations [BETA]\n event Inspect and manage events [BETA]\n issue Manage your issues [BETA]\n mcp Start an MCP server for AI agent access to Hookdeck [BETA]\n metrics Query Event Gateway metrics [BETA]\n request Inspect and manage requests [BETA]\n source Manage your sources [BETA]\n transformation Manage your transformations [BETA]\n\nFlags:\n -h, --help help for gateway\n\nGlobal Flags:\n --color string turn on/off color output (on, off, auto)\n --device-name string device name\n --hookdeck-config string path to CLI config file (default is $HOME/.config/hookdeck/config.toml)\n --insecure Allow invalid TLS certificates\n --log-level string log level (debug, info, warn, error) (default \"info\")\n -p, --profile string profile name (default \"default\")\n\nUse \"hookdeck gateway [command] --help\" for more information about a command.", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "hookdeck gateway issue --help 2>&1", + "description": "Check issue subcommands" + }, + "name": "shell", + "command": "hookdeck gateway issue --help 2>&1", + "result": "Manage Hookdeck issues.\n\nIssues are automatically created when delivery failures, transformation errors,\nor backpressure conditions are detected. Use these commands to list, inspect,\nupdate the status of, or dismiss issues.\n\n[BETA] This feature is in beta. Please share bugs and feedback via:\nhttps://github.com/hookdeck/hookdeck-cli/issues\n\nUsage:\n hookdeck gateway issue [command]\n\nAliases:\n issue, issues\n\nAvailable Commands:\n count Count issues\n dismiss Dismiss an issue\n get Get issue details\n list List issues\n update Update a issue by ID\n\nFlags:\n -h, --help help for issue\n\nGlobal Flags:\n --color string turn on/off color output (on, off, auto)\n --device-name string device name\n --hookdeck-config string path to CLI config file (default is $HOME/.config/hookdeck/config.toml)\n --insecure Allow invalid TLS certificates\n --log-level string log level (debug, info, warn, error) (default \"info\")\n -p, --profile string profile name (default \"default\")\n\nUse \"hookdeck gateway issue [command] --help\" for more information about a command.", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "cat /tmp/sandbox-9ec775af/.claude/skills/event-gateway/references/referencing-docs.md", + "description": "Read how to fetch docs" + }, + "name": "shell", + "command": "cat /tmp/sandbox-9ec775af/.claude/skills/event-gateway/references/referencing-docs.md", + "result": "# Referencing Hookdeck Documentation\n\nHookdeck documentation is the source of truth. Always verify claims against official docs before making Hookdeck-related assertions.\n\n## Fetching Documentation as Markdown\n\nMost Hookdeck doc pages are available as markdown by appending `.md`:\n\n```\ncurl https://hookdeck.com/docs/connections.md\ncurl https://hookdeck.com/docs/filters.md\ncurl https://hookdeck.com/docs/authentication.md\n```\n\n## Use Case Guides\n\nEach use case has an overview and a quickstart:\n\n| Use case | Overview | Quickstart |\n|----------|----------|------------|\n| Receive webhooks | `/docs/use-cases/receive-webhooks.md` | `/docs/use-cases/receive-webhooks/quickstart.md` |\n| Third-party routing | `/docs/use-cases/third-party-routing.md` | `/docs/use-cases/third-party-routing/quickstart.md` |\n| Asynchronous APIs | `/docs/use-cases/asynchronous-apis.md` | `/docs/use-cases/asynchronous-apis/quickstart.md` |\n| Test & debug localhost | `/docs/use-cases/test-debug-localhost.md` | `/docs/use-cases/test-debug-localhost/quickstart.md` |\n\n## Core Reference\n\n| Topic | Path |\n|-------|------|\n| Getting started | `/docs/hookdeck-basics.md` |\n| Sources | `/docs/sources.md` |\n| Destinations | `/docs/destinations.md` |\n| Connections | `/docs/connections.md` |\n| Filters | `/docs/filters.md` |\n| Transformations | `/docs/transformations.md` |\n| Retries | `/docs/retries.md` |\n| Deduplication | `/docs/deduplication.md` |\n| Authentication | `/docs/authentication.md` |\n| Events & Attempts | `/docs/events.md` |\n| Requests | `/docs/requests.md` |\n| Issues | `/docs/issues.md` |\n| Metrics | `/docs/metrics.md` |\n| Bookmarks | `/docs/bookmarks.md` |\n\n## Production (going live)\n\n| Topic | Path | Notes |\n|-------|------|--------|\n| **Receive webhooks → production** | `/docs/use-cases/receive-webhooks/quickstart#deliver-to-your-production-webhook-endpoint` | Deliver to production endpoint; use with Destinations, Retries, Issue triggers for full checklist |\n| **Destinations** | `/docs/destinations` | Set max delivery rate (rate limiting) on destinations |\n| **Retries** | `/docs/retries` | Configure automatic retries |\n| **Issue triggers** | `/docs/issue-triggers` | Define which issues trigger alerts and notifications |\n| **Issues & Notifications** | `/docs/issues` | Notifications, notification channels |\n| **Projects** | `/docs/projects` | Create a new project (e.g. for separate production environment) |\n\nHookdeck docs are the source of truth; point users and agents to these paths for production details.\n\n## CLI Reference\n\n| Page | Path |\n|------|------|\n| CLI overview (about + install + commands) | `/docs/cli.md` |\n| Listen command | `/docs/cli/listen.md` |\n| Connection commands | `/docs/cli/connection.md` |\n| Project commands | `/docs/cli/project.md` |\n| Source commands | `/docs/cli/source.md` |\n| Destination commands | `/docs/cli/destination.md` |\n| Transformation commands | `/docs/cli/transformation.md` |\n| Request commands | `/docs/cli/request.md` |\n| Event commands | `/docs/cli/event.md` |\n| Attempt commands | `/docs/cli/attempt.md` |\n| Metrics commands | `/docs/cli/metrics.md` |\n\n## API Reference\n\nThe API docs are split into focused pages:\n\n| Page | Path |\n|------|------|\n| API overview (auth, pagination, errors, rate limits) | `/docs/api.md` |\n| Connections | `/docs/api/connections.md` |\n| Sources | `/docs/api/sources.md` |\n| Destinations | `/docs/api/destinations.md` |\n| Rules | `/docs/api/rules.md` |\n| Transformations | `/docs/api/transformations.md` |\n| Issue triggers | `/docs/api/issue-triggers.md` |\n| Notifications | `/docs/api/notifications.md` |\n| Publish | `/docs/api/publish.md` |\n| Inspect (requests, events, attempts, bookmarks, issues, metrics) | `/docs/api/inspect.md` |\n| Bulk operations (retry, cancel, ignored events, requests) | `/docs/api/bulk.md` |\n\n## OpenAPI Specification\n\nThe latest Event Gateway API spec:\n\n```\nhttps://api.hookdeck.com/latest/openapi\n```\n\nThe `servers` section contains the current API version.\n\n## Best Practices\n\n1. Don't guess doc URLs -- use the paths listed above\n2. All paths are relative to `https://hookdeck.com`\n3. Docs are the source of truth for syntax, flags, and schemas\n4. This skill provides decision guidance and patterns that docs don't cover", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "curl -s https://hookdeck.com/docs/api/issue-triggers.md", + "description": "Fetch issue triggers API docs" + }, + "name": "shell", + "command": "curl -s https://hookdeck.com/docs/api/issue-triggers.md", + "result": " 404 - Hookdeck\n\n\n

\nThe page you are looking for has been
deleted or does not exist.\n

Home
", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "curl -s https://hookdeck.com/docs/issue-triggers.md 2>&1 | head -300", + "description": "Fetch issue triggers doc page" + }, + "name": "shell", + "command": "curl -s https://hookdeck.com/docs/issue-triggers.md 2>&1 | head -300", + "result": "# Issue Triggers\n\n![Issue Triggers](./images/issue-triggers.png)\n\nIssue triggers are the mechanism within Hookdeck to define which problems should trigger an issue (and a corresponding notification) and under which circumstances.\n\n## Behavior\n\nEffective use of issue triggers relies on an understanding of their behavior and their relationship with the [Issues](/docs/issues) they generate.\n\n### Generated Issues\n\nIssues generated by an issue trigger are internally associated with that issue trigger.\n\n* You can view an issue trigger's associated issues by [inspecting](#inspect-an-issue-trigger) it.\n* [Deleting](#delete-an-issue-trigger) an issue trigger will also delete all associated issues.\n* [Updating](#update-an-issue-trigger) an issue trigger will only affect generated issues after the update. Existing associated issues will be unaffected.\n\n### Notification Channels\n\nSpecify which channels (email, Slack, Discord, Microsoft Teams, Better Stack Uptime, incident.io, etc) should be used to notify you when an issue is triggered. To add a Notification Channel, read [Projects Integrations](/docs/projects#integrations).\n\n### Trigger Overlap\n\nIt is possible to have multiple issue triggers of a given type that overlap in their coverage. In such cases, Hookdeck will attempt to select the most specific trigger and apply its logic.\n\nFor instance, if you had two Delivery triggers and a given connection occurs in the `connections` list of both triggers, the longer of the two lists would be considered more specific and only the more specific trigger would run for that connection. Similarly, a Request trigger configured with a specific rejection cause will take priority over one configured with `'*'` (all causes).\n\n## Issue Types\n\nIssues are currently organized into four types -- delivery issues, transformation issues, backpressure issues, and request issues.\n\n### Delivery Issues\n\nDelivery Issues describe issues related to the attempted delivery of requests, featuring two properties:\n\n* `strategy` : 'first_attempt_failure' or 'last_attempt_failure'\n* `connections` : a list of connections for which the trigger should be applied\n\n### Transformation Issues\n\nTransformation Issues relate to issues that may occur in the data manipulation step of a request:\n\n* `log_level` : the log level where the issue should be opened -- 'warn', 'error', or 'fatal'\n* `transformations` : a list of transformations for which the trigger should be applied\n\n### Backpressure Issues\n\nBackpressure Issues are caused when the estimated queue time (600,000ms, or 10 minutes, by default) exceeds the configured `delay` threshold. This allows for integration-specific alerts for impending queue processing issues.\n\n* `delay` : the queue backpressure, in milliseconds, that should trigger an issue\n* `destinations` : a list of destinations for which the trigger should be applied\n\n### Request Issues\n\nRequest Issues are triggered when incoming webhook requests are rejected by a source. This allows you to monitor and respond to problems with inbound request ingestion before they impact your event pipeline.\n\n* `rejection_causes` : an array of specific rejection causes to match on, or `'*'` to match all rejection causes\n* `sources` : a list of sources for which the trigger should be applied, a pattern to match on source names, or `'*'` for all sources\n\nThe possible rejection causes are:\n\n| Rejection cause | Description |\n| --- | --- |\n| `SOURCE_DISABLED` | The source is disabled and not accepting requests |\n| `NO_CONNECTION` | The source has no connections configured |\n| `VERIFICATION_FAILED` | The request failed the source's verification check |\n| `UNSUPPORTED_HTTP_METHOD` | The request used an HTTP method not supported by the source |\n| `UNSUPPORTED_CONTENT_TYPE` | The request's content type is not supported |\n| `UNPARSABLE_JSON` | The request body contains invalid JSON |\n| `PAYLOAD_TOO_LARGE` | The request payload exceeds the maximum allowed size |\n| `INGESTION_FATAL` | A fatal error occurred during request ingestion |\n| `UNKNOWN` | The request was rejected for an unknown reason |\n\n## Default Triggers\n\nEach project is initialized with one issue trigger of each type:\n\n* The default Delivery issue trigger will alert on the first attempt failure of all connections.\n* The default Transformation issue trigger will alert for events on the `warn` log level of all transformations.\n* The default Backpressure issue trigger will alert when the estimated queue time exceeds the default `delay` of 600000ms (10 minutes) for all destinations.\n* The default Request issue trigger will alert on all rejection causes for all sources.\n\n## View issue triggers\n\nViewing issue triggers gives you an overview of the issue types you'll receive notifications about when they occur in your project. Each issue trigger corresponds to a specific issue type and the details necessary to trigger a notification.\n\n### Dashboard\n1. Open the [Issue Triggers](https://dashboard.hookdeck.com/issue-triggers) page to see a list of all issue triggers, grouped by type.\n\n### API\n`GET /2025-07-01/issue-triggers`\n\n**Response example**\n```json\n{\n \"pagination\": {\n \"order_by\": \"created_at\",\n \"dir\": \"desc\",\n \"limit\": 100\n },\n \"count\": 1,\n \"models\": [\n {\n \"id\": \"it_O9Um6FAQMQvxiP\",\n \"team_id\": \"tm_lbhzBKgFOUnB\",\n \"name\": null,\n \"type\": \"backpressure\",\n \"configs\": {\n \"delay\": 600000,\n \"destinations\": \"*\"\n },\n \"disabled_at\": null,\n \"updated_at\": \"2026-01-14T13:35:52.746Z\",\n \"created_at\": \"2026-01-14T13:35:52.806Z\",\n \"channels\": {\n \"email\": {}\n }\n }\n ]\n}\n```\n\n## Create an issue trigger\n\nCreating an issue trigger tells Hookdeck the circumstances under which an issue notification should be sent.\n\n### Dashboard\n1. Open the [Issues Triggers](https://dashboard.hookdeck.com/issue-triggers) page to see a list of all issue triggers, grouped by type.\n2. Determine the desired trigger type and click the + Create button in the corresponding section.\n3. Change the desired parameters for the given trigger's configuration.\n4. Click the Create Trigger button.\n\n### API\n`POST /2025-07-01/issue-triggers`\n\n**Request body example**\n```json\n{\n \"type\": \"delivery\",\n \"configs\": {\n \"strategy\": \"first_attempt\",\n \"connections\": \"*\"\n },\n \"channels\": {\n \"email\": {}\n }\n}\n```\n\n**Response example**\n```json\n{\n \"type\": \"delivery\",\n \"configs\": {\n \"strategy\": \"first_attempt\",\n \"connections\": \"*\"\n },\n \"channels\": {\n \"email\": {}\n },\n \"id\": \"it_BbDIsKCTbG8qck\",\n \"team_id\": \"tm_lbhzBKgFOUnB\",\n \"created_at\": \"2026-01-14T13:36:41.490Z\",\n \"name\": null,\n \"disabled_at\": null,\n \"updated_at\": \"2026-01-14T13:36:41.491Z\"\n}\n```\n\n## Inspect an issue trigger\n\nInspecting an issue trigger gives you access to its context and activation configuration.\n\n### Dashboard\n1. [Find your issue trigger](#view-issue-triggers) on the Issue Triggers page.\n2. Click the issue trigger's description.From here you can view all issues associated with an issue trigger, [update the parameters](#update-an-issue-trigger) of the selected trigger, and define filters to better target and refine your notification strategy.\n\n### API\n`GET /2025-07-01/issue-triggers/:id`\n\n**Response example**\n```json\n{\n \"id\": \"it_O9Um6FAQMQvxiP\",\n \"team_id\": \"tm_lbhzBKgFOUnB\",\n \"name\": null,\n \"type\": \"backpressure\",\n \"configs\": {\n \"delay\": 600000,\n \"destinations\": \"*\"\n },\n \"disabled_at\": null,\n \"updated_at\": \"2026-01-14T13:35:52.746Z\",\n \"created_at\": \"2026-01-14T13:35:52.806Z\",\n \"channels\": {\n \"email\": {}\n }\n}\n```\n\n## Update an issue trigger\n\nUpdating an issue trigger's configuration lets you define the activation context for when notifications get sent out.\n\n> Updates made to an issue trigger apply only to notifications and issues generated after the update. Issues associated with the trigger prior to any changes will be unaffected.\n\n### Dashboard\n1. [Find your issue trigger](#view-issue-triggers) on the Issue Triggers page.\n2. Click the issue trigger's description to [view the configuration](#inspect-an-issue-trigger).\n3. Change the desired parameters for the given trigger's configuration.\n4. Click Save for the configuration section to save the changes.\n\n### API\n`PUT /2025-07-01/issue-triggers/:id`\n\n**Request body example**\n```json\n{\n \"type\": \"delivery\",\n \"configs\": {\n \"strategy\": \"first_attempt\",\n \"connections\": \"*\"\n },\n \"channels\": {\n \"email\": {}\n }\n}\n```\n\n**Response example**\n```json\n{\n \"id\": \"it_O9Um6FAQMQvxiP\",\n \"team_id\": \"tm_lbhzBKgFOUnB\",\n \"name\": null,\n \"type\": \"backpressure\",\n \"configs\": {\n \"strategy\": \"first_attempt\",\n \"connections\": \"*\"\n },\n \"disabled_at\": null,\n \"updated_at\": \"2026-01-14T13:36:41.518Z\",\n \"created_at\": \"2026-01-14T13:35:52.806Z\",\n \"channels\": {\n \"email\": {}\n }\n}\n```\n\n## Delete an issue trigger\n\nDeleting an issue trigger prevents it from generating notifications.\n\n> Deleting an issue trigger will also permanently delete all associated issues.\n\n### Dashboard\n1. [Find your issue trigger](#view-issue-triggers) on the Issue Triggers page.\n2. Click the issue trigger's description to [view its configuration](#inspect-an-issue-trigger).\n3. Click Delete at the bottom of the page.\n4. Click Delete again to confirm.\n\n### API\n`DELETE /2025-07-01/issue-triggers/:id`\n\n**Response example**\n```json\n{\n \"id\": \"it_O9Um6FAQMQvxiP\"\n}\n```", + "ts": 0 + }, + { + "endpoint": "Bash", + "body": { + "command": "echo \"=== existing issue triggers ===\"; curl -s https://api.hookdeck.com/2025-07-01/issue-triggers -H \"Authorization: Bearer $HOOKDECK_API_KEY\" | head -c 3000\necho\necho \"=== notifications docs ===\"\ncurl -s https://hookdeck.com/docs/api/notifications.md 2>&1 | head -c 100\necho\necho \"=== issues doc channels section ===\"\ncurl -s https://hookdeck.com/docs/issues.md 2>&1 | grep -n -i -A5 \"channel\"", + "description": "Check existing issue triggers and notification docs" + }, + "name": "shell", + "command": "echo \"=== existing issue triggers ===\"; curl -s https://api.hookdeck.com/2025-07-01/issue-triggers -H \"Authorization: Bearer $HOOKDECK_API_KEY\" | head -c 3000\necho\necho \"=== notifications docs ===\"\ncurl -s https://hookdeck.com/docs/api/notifications.md 2>&1 | head -c 100\necho\necho \"=== issues doc channels section ===\"\ncurl -s https://hookdeck.com/docs/issues.md 2>&1 | grep -n -i -A5 \"channel\"", + "result": "=== existing issue triggers ===\n{\"pagination\":{\"order_by\":\"created_at\",\"dir\":\"desc\",\"limit\":100},\"count\":3,\"models\":[{\"id\":\"it_ayecSXxpWrAxfC\",\"team_id\":\"tm_c6MTDfxZm4tP\",\"name\":null,\"type\":\"transformation\",\"configs\":{\"log_level\":\"warn\",\"transformations\":\"*\"},\"disabled_at\":null,\"updated_at\":\"2026-08-07T10:46:14.027Z\",\"created_at\":\"2026-08-07T10:46:14.347Z\",\"channels\":{\"email\":{}}},{\"id\":\"it_MG6NeNJn9TxeRa\",\"team_id\":\"tm_c6MTDfxZm4tP\",\"name\":null,\"type\":\"request\",\"configs\":{\"sources\":\"*\",\"rejection_causes\":\"*\",\"excluded_rejection_causes\":[\"NO_CONNECTION\",\"UNSUPPORTED_HTTP_METHOD\"]},\"disabled_at\":null,\"updated_at\":\"2026-08-07T10:46:14.027Z\",\"created_at\":\"2026-08-07T10:46:14.347Z\",\"channels\":{\"email\":{}}},{\"id\":\"it_T7Z6hXLnlhazwM\",\"team_id\":\"tm_c6MTDfxZm4tP\",\"name\":null,\"type\":\"backpressure\",\"configs\":{\"delay\":600000,\"destinations\":\"*\"},\"disabled_at\":null,\"updated_at\":\"2026-08-07T10:46:14.027Z\",\"created_at\":\"2026-08-07T10:46:14.347Z\",\"channels\":{\"email\":{}}}]}\n=== notifications docs ===\n 404 - Hookdeck

Issue Triggers

\"Issue

Issue triggers are the mechanism within Hookdeck to define which problems should trigger an issue (and a corresponding notification) and under which circumstances.

Behavior

Effective use of issue triggers relies on an understanding of their behavior and their relationship with the Issues they generate.

Generated Issues

Issues generated by an issue trigger are internally associated with that issue trigger.

  • You can view an issue trigger's associated issues by inspecting it.
  • Deleting an issue trigger will also delete all associated issues.
  • Updating an issue trigger will only affect generated issues after the update. Existing associated issues will be unaffected.

Notification Channels

Specify which channels (email, Slack, Discord, Microsoft Teams, Better Stack Uptime, incident.io, etc) should be used to notify you when an issue is triggered. To add a Notification Channel, read Projects Integrations.

Trigger Overlap

It is possible to have multiple issue triggers of a given type that overlap in their coverage. In such cases, Hookdeck will attempt to select the most specific trigger and apply its logic.

For instance, if you had two Delivery triggers and a given connection occurs in the connections list of both triggers, the longer of the two lists would be considered more specific and only the more specific trigger would run for that connection. Similarly, a Request trigger configured with a specific rejection cause will take priority over one configured with '*' (all causes).

Issue Types

Issues are currently organized into four types -- delivery issues, transformation issues, backpressure issues, and request issues.

Delivery Issues

Delivery Issues describe issues related to the attempted delivery of requests, featuring two properties:

  • strategy : 'first_attempt_failure' or 'last_attempt_failure'
  • connections : a list of connections for which the trigger should be applied

Transformation Issues

Transformation Issues relate to issues that may occur in the data manipulation step of a request:

  • log_level : the log level where the issue should be opened -- 'warn', 'error', or 'fatal'
  • transformations : a list of transformations for which the trigger should be applied

Backpressure Issues

Backpressure Issues are caused when the estimated queue time (600,000ms, or 10 minutes, by default) exceeds the configured delay threshold. This allows for integration-specific alerts for impending queue processing issues.

  • delay : the queue backpressure, in milliseconds, that should trigger an issue
  • destinations : a list of destinations for which the trigger should be applied

Request Issues

Request Issues are triggered when incoming webhook requests are rejected by a source. This allows you to monitor and respond to problems with inbound request ingestion before they impact your event pipeline.

  • rejection_causes : an array of specific rejection causes to match on, or '*' to match all rejection causes
  • sources : a list of sources for which the trigger should be applied, a pattern to match on source names, or '*' for all sources

The possible rejection causes are:

Rejection causeDescription
SOURCE_DISABLEDThe source is disabled and not accepting requests
NO_CONNECTIONThe source has no connections configured
VERIFICATION_FAILEDThe request failed the source's verification check
UNSUPPORTED_HTTP_METHODThe request used an HTTP method not supported by the source
UNSUPPORTED_CONTENT_TYPEThe request's content type is not supported
UNPARSABLE_JSONThe request body contains invalid JSON
PAYLOAD_TOO_LARGEThe request payload exceeds the maximum allowed size
INGESTION_FATALA fatal error occurred during request ingestion
UNKNOWNThe request was rejected for an unknown reason

Default Triggers

Each project is initialized with one issue trigger of each type:

  • The default Delivery issue trigger will alert on the first attempt failure of all connections.
  • The default Transformation issue trigger will alert for events on the warn log level of all transformations.
  • The default Backpressure issue trigger will alert when the estimated queue time exceeds the default delay of 600000ms (10 minutes) for all destinations.
  • The default Request issue trigger will alert on all rejection causes for all sources.

View issue triggers

Viewing issue triggers gives you an overview of the issue types you'll receive notifications about when they occur in your project. Each issue trigger corresponds to a specific issue type and the details necessary to trigger a notification.

  1. Open the Issue Triggers page to see a list of all issue triggers, grouped by type.
GET
/2025-07-01/issue-triggers
Response example
JSON 
{\n295:  \"type\": \"delivery\",\n297:    \"strategy\": \"first_attempt\",\n298:    \"connections\": \"*\"\n304:  \"type\": \"delivery\",\n306:    \"strategy\": \"first_attempt\",\n307:    \"connections\": \"*\"\n318:}

Inspect an issue trigger

Inspecting an issue trigger gives you access to its context and activation configuration.

  1. Find your issue trigger on the Issue Triggers page.
  2. Click the issue trigger's description.

From here you can view all issues associated with an issue trigger, update the parameters of the selected trigger, and define filters to better target and refine your notification strategy.

GET
/2025-07-01/issue-triggers/:id
Response example
JSON 
{\n334:  \"type\": \"delivery\",\n336:    \"strategy\": \"first_attempt\",\n337:    \"connections\": \"*\"\n348:    \"strategy\": \"first_attempt\",\n349:    \"connections\": \"*\"\n359:}
<- Previous Issues & Notifications Next -> Metrics