From 2899c4e0b78d59f23a14b227387bcf66c2bffa0f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:31:22 +0000 Subject: [PATCH] feat(service): Add Sentry spans and cache metrics Each proxied request is now a trace, with spans for the key, the store read, and the live request. New metrics count each request by result (replayed, live, missed, passthrough) and time it, so the hit rate of each tenant and rule shows in Sentry. They replace roach.recording, and they do not have the key or the run. The SDK no longer adds request data or trace headers: request data had URLs and bodies, and the proxy sent sentry-trace and baggage upstream. Co-Authored-By: David Cramer --- README.md | 54 ++++++++++++- src/recorder.ts | 110 +++++++++++++++++++++---- src/server.ts | 27 +++++-- src/service.ts | 136 ++++++++++++++++++++++++------- tests/deployed.test.ts | 178 +++++++++++++++++++++++++++++++++++------ 5 files changed, 429 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 65a4396..094a539 100644 --- a/README.md +++ b/README.md @@ -219,14 +219,60 @@ first miss of each test, which is the one to fix. `valuePatterns` of the config. Check each one for slow backtracking before you add it. - **Sentry.** With `sentryDsn`, each read and write of a recording sends - the metric `roach.recording`. Its attributes are `tenant`, `rule`, `key`, - `run` (the `name` of the run), and `result`: `replayed`, `missed`, or - `written`. Group by `key` and count `run`, and a recording that only one - run uses shows that a value changes on each run. + a span to Sentry. Each proxied request sends a span and metrics. See + [Sentry](#sentry). - **Certificate authority.** `ca` holds the PEM certificate and key that clients trust. Keep it across restarts. Host certificates live for 7 days, and the service makes new ones before they expire. +### Sentry + +With `sentryDsn`, the service sends metrics and spans to Sentry. It keeps +all traces. Metrics have only attributes with few values: `tenant`, `mode`, +`rule`, `origin`, and `result`. Spans also have `run` (the `name` of the +run) and `key`. + +| Metric | Type | Value | +| ------------------------ | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `roach.request` | counter | 1 for each proxied request. `result` is the `x-roach` header: `replayed`, `live`, `missed`, or `passthrough`. A `passthrough` request has no `rule`. | +| `roach.request.duration` | distribution (ms) | The time to answer a request that a rule records, by `result`. | +| `roach.recording.write` | counter | Recordings by `rule`. `result` is `written` (the proxy sent them to the store) or `dropped` (their session failed). | + +Each proxied request is one trace. Each span has `tenant`, `mode`, and +`run`. + +- `roach.run.start`: a new run, with `write` (the run has the write token). +- `roach.request`: one proxied request, named ` `, with + `origin`, `rule`, `result`, and `http.response.status_code`. It contains: + - `roach.key`: the key of the request, with `rule` and `key`. + - `roach.store.read`: a read of a recording, with `key` and `result`: + `hit` or `miss`. + - `roach.upstream`: the live request, with `origin` and + `http.response.status_code`. +- `roach.store.write`: a write of recordings, with `recordings` and + `written`. A session writes when it ends, so this span is usually its + own trace. + +Telemetry never has a credential, a body, or the path or query of a URL. +So the service turns off the request data and the HTTP spans of the SDK, +and it adds no trace headers to the requests that it sends. + +To see how well the cache works: + +- **Hit rate.** Sum `roach.request`, group by `result`, and filter by + `tenant` and `rule`. The hit rate is `replayed / (replayed + live + + missed)`. +- **Misses.** `missed` is a request that failed in `replay` mode. `live` + in `auto` mode is a new recording. When `live` stays high from run to + run, a value in the key changes on each run. +- **Time.** Compare `roach.request.duration` for `live` and `replayed`. +- **Dropped.** `roach.recording.write` with `result:dropped` counts + recordings of failed sessions. The next run sends these requests live + again. +- **Changing keys.** Search spans for `span.op:roach.key`, group by + `key`, and count the runs. A key that only one run uses shows that a + value changes on each run. + ### Limits - Runs live in memory in one process. A restart ends the open runs, and diff --git a/src/recorder.ts b/src/recorder.ts index d876c9e..af465d8 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -14,7 +14,8 @@ * its session ended follows the result of that session. A request outside * a session is written at once. * - * The recorder never writes a known credential (`secrets.ts`). + * The recorder never writes a known credential (`secrets.ts`). It reports + * spans and metrics through the `Telemetry` hooks of `service.ts`. */ import type { IncomingHttpHeaders } from "node:http"; import { keyRequest, type KeyedRequest } from "./request-key.ts"; @@ -56,6 +57,29 @@ export interface ProxyResponse { /** Where the response to a recorded request came from. */ export type ResponseSource = "live" | "missed" | "replayed"; +/** The attributes of a span or a metric. */ +type Attributes = Record; + +/** + * Hooks that report spans and metrics. `service.ts` gives hooks that send + * them to Sentry, so this file and `server.ts` do not load the SDK. An + * attribute must never hold a credential, a body, or the path or query of + * a URL. + */ +export interface Telemetry { + /** Run `fn` in a span. The span ends when `fn` returns or settles. */ + span( + options: { op: string; name?: string; attributes?: Attributes }, + fn: () => T, + ): T; + /** Add attributes to the active span. */ + annotate(attributes: Attributes): void; + /** Add `value` to the counter `name`. */ + count(name: string, value: number, attributes: Attributes): void; + /** Add a time in milliseconds to the distribution `name`. */ + time(name: string, ms: number, attributes: Attributes): void; +} + interface Session { name: string; /** New recordings, by key. */ @@ -142,6 +166,7 @@ export interface RecorderConfig { rules: RecordingRule[]; /** Credentials to redact, in addition to the ones that it learns. */ secrets?: string[]; + telemetry: Telemetry; } /** Create the recorder of one run, with the store of its recordings. */ @@ -152,6 +177,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } } let session: Session | undefined; + const { telemetry } = config; const secrets = createSecrets(config.secrets); const stats: RecordingStats = { counts: Object.fromEntries( @@ -165,8 +191,27 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { discarded: 0, passthrough: {}, }; + /** Count recordings by rule in `roach.recording.write`. */ + const countRecordings = (keys: string[], result: "dropped" | "written") => { + const byRule = new Map(); + for (const key of keys) { + const rule = key.slice(0, key.indexOf("/")); + byRule.set(rule, (byRule.get(rule) ?? 0) + 1); + } + for (const [rule, value] of byRule) { + telemetry.count("roach.recording.write", value, { rule, result }); + } + }; const write = async (recordings: Array<[string, Recording]>) => { stats.written += await store.write(recordings); + countRecordings( + recordings.map(([key]) => key), + "written", + ); + }; + const drop = (keys: string[]) => { + stats.discarded += keys.length; + countRecordings(keys, "dropped"); }; const record = async ( @@ -177,7 +222,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { if (owner && owner.passed === undefined) { owner.recorded.set(key, recording); } else if (owner?.passed === false) { - stats.discarded += 1; + drop([key]); } else { await write([[key, recording]]); } @@ -201,7 +246,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { session = undefined; ended.passed = passed; if (passed) await write([...ended.recorded]); - else stats.discarded += ended.recorded.size; + else drop([...ended.recorded.keys()]); return ended.missed; }; @@ -216,6 +261,8 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { countPassthrough(origin: string, headers: IncomingHttpHeaders): void { secrets.learn(headers); stats.passthrough[origin] = (stats.passthrough[origin] ?? 0) + 1; + telemetry.annotate({ result: "passthrough" }); + telemetry.count("roach.request", 1, { origin, result: "passthrough" }); }, /** @@ -228,19 +275,30 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { request: ProxyRequest, sendLive: () => Promise, ): Promise { + const started = performance.now(); + const origin = request.url.origin; // A request belongs to the session that was open when it started. const owner = session; const counts = stats.counts[rule.name]!; secrets.learn(request.headers); - const keyed = keyRequest( - rule, - { - method: request.method, - url: request.url.href, - headers: request.headers, - body: request.body.toString("utf8"), + telemetry.annotate({ rule: rule.name }); + const { keyed, key } = telemetry.span( + { op: "roach.key", attributes: { rule: rule.name } }, + () => { + const result = keyRequest( + rule, + { + method: request.method, + url: request.url.href, + headers: request.headers, + body: request.body.toString("utf8"), + }, + owner?.known, + ); + const name = recordingKey(rule.name, result.key); + telemetry.annotate({ key: name }); + return { keyed: result, key: name }; }, - owner?.known, ); if (owner) { for (const [name, list] of keyed.values) { @@ -249,12 +307,22 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } } } - const key = recordingKey(rule.name, keyed.key); + const report = (result: ResponseSource) => { + const attributes = { rule: rule.name, origin, result }; + telemetry.annotate({ result }); + telemetry.count("roach.request", 1, attributes); + telemetry.time( + "roach.request.duration", + performance.now() - started, + attributes, + ); + }; if (config.mode !== "record") { const recording = await store.read(key); if (recording) { counts.replayed += 1; + report("replayed"); return { ...fromRecording(recording, keyed.values), source: "replayed", @@ -264,6 +332,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { if (config.mode === "replay") { counts.missed += 1; if (owner) owner.missed += 1; + report("missed"); return { status: 412, headers: { "content-type": "text/plain" }, @@ -276,18 +345,29 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } counts.live += 1; + const live = await telemetry.span( + { + op: "roach.upstream", + name: `${request.method} ${origin}`, + attributes: { origin, "http.request.method": request.method }, + }, + async () => { + const response = await sendLive(); + telemetry.annotate({ "http.response.status_code": response.status }); + return response; + }, + ); // The client gets the redacted response too, so a live run and a // replay give the same body. const recording = JSON.parse( secrets.redact( - JSON.stringify( - toRecording(request, keyed, owner?.name, await sendLive()), - ), + JSON.stringify(toRecording(request, keyed, owner?.name, live)), ), ) as Recording; if (!isTemporaryStatus(recording.response.status)) { await record(owner, key, recording); } + report("live"); return { ...fromRecording(recording, keyed.values), source: "live" }; }, diff --git a/src/server.ts b/src/server.ts index c46b12b..f1e172e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,7 +20,7 @@ import https from "node:https"; import { isIP, type Socket } from "node:net"; import tls from "node:tls"; import type { CertificateAuthority } from "./certificates.ts"; -import type { ProxyRequest, Recorder } from "./recorder.ts"; +import type { ProxyRequest, Recorder, Telemetry } from "./recorder.ts"; /** The path prefix of the control API. */ export const CONTROL_PATH = "/__roach"; @@ -289,6 +289,8 @@ export interface ProxyTarget { replayOnly?: boolean; /** Set when the target stops. Its tunnels then refuse new requests. */ closed?: boolean; + /** Gets a span for each proxied request. */ + telemetry: Telemetry; } /** A target, or the HTTP status that refuses the request. */ @@ -345,8 +347,24 @@ function serveProxied( outgoing.end("Roach: this run has ended\n"); return; } - proxyRequest(target, incoming, outgoing, tunnelOrigin).catch( - (error: unknown) => { + // Only the origin: the path and query can hold a credential. + const origin = tunnelOrigin ?? URL.parse(incoming.url ?? "")?.origin; + const method = incoming.method ?? "GET"; + target.telemetry + .span( + { + op: "roach.request", + name: `${method} ${origin}`, + attributes: { origin: String(origin), "http.request.method": method }, + }, + async () => { + await proxyRequest(target, incoming, outgoing, tunnelOrigin); + target.telemetry.annotate({ + "http.response.status_code": outgoing.statusCode, + }); + }, + ) + .catch((error: unknown) => { if (!outgoing.headersSent) { const status = (error as { status?: number }).status ?? 502; outgoing.writeHead(status, { "content-type": "text/plain" }); @@ -354,8 +372,7 @@ function serveProxied( outgoing.end( `Roach error: ${error instanceof Error ? error.message : String(error)}\n`, ); - }, - ); + }); } /** diff --git a/src/service.ts b/src/service.ts index 415289d..13bbbaa 100644 --- a/src/service.ts +++ b/src/service.ts @@ -19,8 +19,9 @@ * - The recordings of a tenant are under `/`, in a directory or a * GCS bucket (`gcs.ts`). Tenants never share recordings, because a key * does not include credentials. - * - Each read and write of a recording sends the Sentry metric - * `roach.recording`, with the tenant, the key, and the run. + * - With `sentryDsn`, the service sends spans and metrics to Sentry. See + * "Sentry" in `README.md`. Metrics have only attributes with few values, + * so the key and the run are only on spans. */ import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; import type http from "node:http"; @@ -31,7 +32,7 @@ import { type CertificateAuthorityPem, } from "./certificates.ts"; import { createGcsStore } from "./gcs.ts"; -import { createRecorder } from "./recorder.ts"; +import { createRecorder, type Telemetry } from "./recorder.ts"; import { createFileStore } from "./recordings.ts"; import { CONTROL_PATH, @@ -141,6 +142,8 @@ interface Run extends ProxyTarget { } const MODES = new Set(["auto", "off", "record", "replay"]); +/** The service gets little traffic, so it keeps all traces. */ +const TRACES_SAMPLE_RATE = 1; /** A run that is open longer than this ends as failed, such as a dead job. */ const RUN_TIMEOUT_MS = 6 * 60 * 60 * 1000; /** @@ -249,36 +252,73 @@ function invalidRunConfig( } /** - * The store of one run. It counts each read and write in Sentry, and a - * run without write access cannot write. + * The telemetry of one run, in Sentry. Metrics get `labels`. Spans also get + * the name of the run. + */ +function runTelemetry( + labels: { tenant: string; mode: RecordingMode }, + run: string, +): Telemetry { + return { + span: ({ op, name = op, attributes }, fn) => + Sentry.startSpan( + { op, name, attributes: { ...labels, run, ...attributes } }, + fn, + ), + annotate: (attributes) => { + Sentry.getActiveSpan()?.setAttributes(attributes); + }, + count: (name, value, attributes) => { + Sentry.metrics.count(name, value, { + attributes: { ...labels, ...attributes }, + }); + }, + time: (name, ms, attributes) => { + Sentry.metrics.distribution(name, ms, { + unit: "millisecond", + attributes: { ...labels, ...attributes }, + }); + }, + }; +} + +/** + * The store of one run. Each read and write has a span, and a run without + * write access cannot write. */ function runStore( base: RecordingStore, - labels: { tenant: string; run: string }, + labels: { tenant: string; mode: RecordingMode; run: string }, canWrite: boolean, ): RecordingStore { - const count = (key: string, result: string) => { - Sentry.metrics.count("roach.recording", 1, { - attributes: { - ...labels, - rule: key.slice(0, key.indexOf("/")), - key, - result, - }, - }); - }; + const span = ( + op: string, + attributes: Record, + fn: (span: Sentry.Span) => Promise, + ) => + Sentry.startSpan( + { op, name: op, attributes: { ...labels, ...attributes } }, + fn, + ); return { - async read(key) { - const recording = await base.read(key); - count(key, recording ? "replayed" : "missed"); - return recording; - }, + read: (key) => + span("roach.store.read", { key }, async (current) => { + const recording = await base.read(key); + current.setAttribute("result", recording ? "hit" : "miss"); + return recording; + }), async write(recordings) { if (recordings.length === 0) return 0; if (!canWrite) throw new Error("This run cannot write recordings"); - const written = await base.write(recordings); - for (const [key] of recordings) count(key, "written"); - return written; + return span( + "roach.store.write", + { recordings: recordings.length }, + async (current) => { + const written = await base.write(recordings); + current.setAttribute("written", written); + return written; + }, + ); }, }; } @@ -295,7 +335,26 @@ export async function startRoachService( } const writeTokenHash = Buffer.from(config.writeTokenHash, "hex"); if (config.sentryDsn) { - Sentry.init({ dsn: config.sentryDsn, tracesSampleRate: 0 }); + Sentry.init({ + dsn: config.sentryDsn, + tracesSampleRate: TRACES_SAMPLE_RATE, + // Send only what Roach puts on its spans. The HTTP integrations and + // `RequestData` add whole URLs, headers, and bodies, which can hold + // credentials, such as the `secrets` of a run. And the proxy must not + // add trace headers to the requests that it sends. + tracePropagationTargets: [], + integrations: (defaults) => [ + ...defaults.filter( + ({ name }) => !["Http", "NodeFetch", "RequestData"].includes(name), + ), + Sentry.httpIntegration({ + spans: false, + breadcrumbs: false, + maxRequestBodySize: "none", + }), + Sentry.nativeNodeFetchIntegration({ spans: false, breadcrumbs: false }), + ], + }); } const serviceOrigins = parseOrigins(config.allow); const runs = new Map(); @@ -319,11 +378,19 @@ export async function startRoachService( const createRun = (runConfig: RunConfig, canWrite: boolean): RemoteRun => { const id = randomBytes(8).toString("hex"); + const telemetry = runTelemetry( + { tenant: runConfig.tenant, mode: runConfig.mode }, + runConfig.name ?? id, + ); const token = randomBytes(24).toString("hex"); const allow = runConfig.allow ?? config.allow; const store = runStore( storeOf(runConfig.tenant), - { tenant: runConfig.tenant, run: runConfig.name ?? id }, + { + tenant: runConfig.tenant, + mode: runConfig.mode, + run: runConfig.name ?? id, + }, canWrite, ); const run: Run = { @@ -339,9 +406,11 @@ export async function startRoachService( mode: runConfig.mode, rules: runConfig.rules, secrets: [...(config.secrets ?? []), ...(runConfig.secrets ?? [])], + telemetry, }, store, ), + telemetry, }; runs.set(id, run); process.stderr.write( @@ -415,7 +484,20 @@ export async function startRoachService( return; } } - sendJson(outgoing, 201, createRun(runConfig as RunConfig, canWrite)); + const valid = runConfig as RunConfig; + const remote = Sentry.startSpan( + { + op: "roach.run.start", + name: "roach.run.start", + attributes: { tenant, mode: valid.mode, write: canWrite }, + }, + (span) => { + const created = createRun(valid, canWrite); + span.setAttribute("run", valid.name ?? created.id); + return created; + }, + ); + sendJson(outgoing, 201, remote); return; } diff --git a/tests/deployed.test.ts b/tests/deployed.test.ts index 1ba5b1a..0b36bf7 100644 --- a/tests/deployed.test.ts +++ b/tests/deployed.test.ts @@ -14,7 +14,7 @@ import { execFile, spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; import { once } from "node:events"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { createServer } from "node:http"; +import { createServer, type IncomingHttpHeaders } from "node:http"; import https from "node:https"; import net, { type AddressInfo } from "node:net"; import { tmpdir } from "node:os"; @@ -35,7 +35,16 @@ let directory: string; let hostCertFile: string; let caCert: string; const objects = new Map(); -const metrics: Array> = []; +const envelopes: string[] = []; +/** A metric or a span that the service sent to Sentry. */ +interface TelemetryItem { + name: string; + value?: number; + attributes: Record; +} +const metrics: TelemetryItem[] = []; +const spans: TelemetryItem[] = []; +const upstreamHeaders: IncomingHttpHeaders[] = []; let liveRequests = 0; let upstreamOrigin: string; let frontUrl: string; @@ -59,23 +68,58 @@ async function listen(server: net.Server, host = "127.0.0.1") { return (server.address() as AddressInfo).port; } -/** Keep the `roach.recording` metrics of a Sentry envelope. */ +/** + * Keep the metrics and the spans of a Sentry envelope. Both are lists of + * items with typed attributes. + */ function readEnvelope(envelope: string) { - for (const line of envelope.split("\n")) { - if (!line.includes('"roach.recording"')) continue; - const { items } = JSON.parse(line) as { - items: Array<{ attributes: Record }>; + envelopes.push(envelope); + // A header line, then a header line and a payload line for each item. + const [, ...lines] = envelope.split("\n").filter(Boolean); + for (let index = 0; index + 1 < lines.length; index += 2) { + const { type } = JSON.parse(lines[index]!) as { type: string }; + const list = { trace_metric: metrics, span: spans }[type]; + if (!list) continue; + const { items } = JSON.parse(lines[index + 1]!) as { + items: Array< + Omit & { + attributes: Record; + } + >; }; - for (const { attributes } of items) { - metrics.push( - Object.fromEntries( - Object.entries(attributes).map(([name, { value }]) => [name, value]), + for (const item of items) { + list.push({ + ...item, + attributes: Object.fromEntries( + Object.entries(item.attributes).map(([name, { value }]) => [ + name, + value, + ]), ), - ); + }); } } } +/** The metrics of one request of the `model` rule. */ +function requestMetrics(mode: string, result: string) { + const attributes = { + tenant: TENANT, + mode, + rule: "model", + origin: upstreamOrigin, + result, + }; + return [ + { name: "roach.request", value: 1, attributes }, + { + name: "roach.request.duration", + value: expect.any(Number), + attributes, + }, + ]; +} + /** * Run one CI job: the action (`src/action.ts`) with a command that sends * one HTTPS request with plain `fetch`. Returns the exit code of the @@ -197,8 +241,9 @@ beforeAll(async () => { }; // The upstream API. It is not on `localhost`, so `NO_PROXY` keeps it. - const upstream = https.createServer(host, (_incoming, outgoing) => { + const upstream = https.createServer(host, (incoming, outgoing) => { liveRequests += 1; + upstreamHeaders.push(incoming.headers); outgoing.writeHead(200, { "content-type": "application/json" }); outgoing.end(JSON.stringify({ n: liveRequests })); }); @@ -361,23 +406,106 @@ describe("deployed roach", () => { expect(await once(service, "exit")).toEqual([0, null]); const [key] = objects.keys(); const relativeKey = key!.slice(`${TENANT}/`.length); + + // Metrics have only attributes with few values: no key and no run. + const labels = ["tenant", "mode", "rule", "origin", "result", "key", "run"]; expect( - metrics.map(({ tenant, run, result, key: metricKey }) => ({ - tenant, - run, - result, - key: metricKey, + metrics.map(({ name, value, attributes }) => ({ + name, + value, + attributes: Object.fromEntries( + labels.flatMap((label) => + label in attributes ? [[label, attributes[label]]] : [], + ), + ), })), ).toEqual([ - { tenant: TENANT, run: "writer-1", result: "missed", key: relativeKey }, - { tenant: TENANT, run: "writer-1", result: "written", key: relativeKey }, - { tenant: TENANT, run: "fork-1", result: "replayed", key: relativeKey }, + ...requestMetrics("auto", "live"), { - tenant: TENANT, - run: "miss-1", - result: "missed", - key: expect.stringMatching(/^model\//), + name: "roach.recording.write", + value: 1, + attributes: { + tenant: TENANT, + mode: "auto", + rule: "model", + result: "written", + }, }, + ...requestMetrics("replay", "replayed"), + ...requestMetrics("replay", "missed"), ]); + + // Only the spans of Roach: 4 for each run, and 2 for the live request. + const ops = spans.map(({ attributes }) => attributes["sentry.op"]); + expect(ops).toHaveLength(14); + expect(new Set(ops)).toEqual( + new Set([ + "roach.run.start", + "roach.request", + "roach.key", + "roach.store.read", + "roach.upstream", + "roach.store.write", + ]), + ); + const span = (op: string, run: string) => + spans.find( + ({ attributes }) => + attributes["sentry.op"] === op && attributes.run === run, + )?.attributes; + expect(span("roach.run.start", "writer-1")).toMatchObject({ + tenant: TENANT, + mode: "auto", + write: true, + }); + expect(span("roach.request", "writer-1")).toMatchObject({ + tenant: TENANT, + mode: "auto", + rule: "model", + origin: upstreamOrigin, + result: "live", + "http.response.status_code": 200, + }); + expect(span("roach.request", "fork-1")).toMatchObject({ + result: "replayed", + }); + expect(span("roach.request", "miss-1")).toMatchObject({ + result: "missed", + "http.response.status_code": 412, + }); + expect(span("roach.key", "writer-1")).toMatchObject({ key: relativeKey }); + expect(span("roach.store.read", "writer-1")).toMatchObject({ + key: relativeKey, + result: "miss", + }); + expect(span("roach.store.read", "fork-1")).toMatchObject({ + key: relativeKey, + result: "hit", + }); + expect(span("roach.upstream", "writer-1")).toMatchObject({ + origin: upstreamOrigin, + "http.response.status_code": 200, + }); + expect(span("roach.store.write", "writer-1")).toMatchObject({ + recordings: 1, + written: 1, + }); + + // Sentry gets no credential, no path, and no body, and the upstream + // gets no trace headers. + for (const secret of [ + TOKEN, + GCS_TOKEN, + "/v1/messages", + "/__roach/", + "prompt", + ]) { + expect(envelopes.join("\n")).not.toContain(secret); + } + for (const headers of upstreamHeaders) { + expect(headers).not.toHaveProperty("sentry-trace"); + expect(headers).not.toHaveProperty("baggage"); + expect(headers).not.toHaveProperty("traceparent"); + } }); });