diff --git a/README.md b/README.md index 65a4396..094a539 100644 --- a/README.md +++ b/README.md @@ -219,14 +219,60 @@ first miss of each test, which is the one to fix. `valuePatterns` of the config. Check each one for slow backtracking before you add it. - **Sentry.** With `sentryDsn`, each read and write of a recording sends - the metric `roach.recording`. Its attributes are `tenant`, `rule`, `key`, - `run` (the `name` of the run), and `result`: `replayed`, `missed`, or - `written`. Group by `key` and count `run`, and a recording that only one - run uses shows that a value changes on each run. + a span to Sentry. Each proxied request sends a span and metrics. See + [Sentry](#sentry). - **Certificate authority.** `ca` holds the PEM certificate and key that clients trust. Keep it across restarts. Host certificates live for 7 days, and the service makes new ones before they expire. +### Sentry + +With `sentryDsn`, the service sends metrics and spans to Sentry. It keeps +all traces. Metrics have only attributes with few values: `tenant`, `mode`, +`rule`, `origin`, and `result`. Spans also have `run` (the `name` of the +run) and `key`. + +| Metric | Type | Value | +| ------------------------ | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `roach.request` | counter | 1 for each proxied request. `result` is the `x-roach` header: `replayed`, `live`, `missed`, or `passthrough`. A `passthrough` request has no `rule`. | +| `roach.request.duration` | distribution (ms) | The time to answer a request that a rule records, by `result`. | +| `roach.recording.write` | counter | Recordings by `rule`. `result` is `written` (the proxy sent them to the store) or `dropped` (their session failed). | + +Each proxied request is one trace. Each span has `tenant`, `mode`, and +`run`. + +- `roach.run.start`: a new run, with `write` (the run has the write token). +- `roach.request`: one proxied request, named ` `, with + `origin`, `rule`, `result`, and `http.response.status_code`. It contains: + - `roach.key`: the key of the request, with `rule` and `key`. + - `roach.store.read`: a read of a recording, with `key` and `result`: + `hit` or `miss`. + - `roach.upstream`: the live request, with `origin` and + `http.response.status_code`. +- `roach.store.write`: a write of recordings, with `recordings` and + `written`. A session writes when it ends, so this span is usually its + own trace. + +Telemetry never has a credential, a body, or the path or query of a URL. +So the service turns off the request data and the HTTP spans of the SDK, +and it adds no trace headers to the requests that it sends. + +To see how well the cache works: + +- **Hit rate.** Sum `roach.request`, group by `result`, and filter by + `tenant` and `rule`. The hit rate is `replayed / (replayed + live + + missed)`. +- **Misses.** `missed` is a request that failed in `replay` mode. `live` + in `auto` mode is a new recording. When `live` stays high from run to + run, a value in the key changes on each run. +- **Time.** Compare `roach.request.duration` for `live` and `replayed`. +- **Dropped.** `roach.recording.write` with `result:dropped` counts + recordings of failed sessions. The next run sends these requests live + again. +- **Changing keys.** Search spans for `span.op:roach.key`, group by + `key`, and count the runs. A key that only one run uses shows that a + value changes on each run. + ### Limits - Runs live in memory in one process. A restart ends the open runs, and diff --git a/src/recorder.ts b/src/recorder.ts index d876c9e..af465d8 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -14,7 +14,8 @@ * its session ended follows the result of that session. A request outside * a session is written at once. * - * The recorder never writes a known credential (`secrets.ts`). + * The recorder never writes a known credential (`secrets.ts`). It reports + * spans and metrics through the `Telemetry` hooks of `service.ts`. */ import type { IncomingHttpHeaders } from "node:http"; import { keyRequest, type KeyedRequest } from "./request-key.ts"; @@ -56,6 +57,29 @@ export interface ProxyResponse { /** Where the response to a recorded request came from. */ export type ResponseSource = "live" | "missed" | "replayed"; +/** The attributes of a span or a metric. */ +type Attributes = Record; + +/** + * Hooks that report spans and metrics. `service.ts` gives hooks that send + * them to Sentry, so this file and `server.ts` do not load the SDK. An + * attribute must never hold a credential, a body, or the path or query of + * a URL. + */ +export interface Telemetry { + /** Run `fn` in a span. The span ends when `fn` returns or settles. */ + span( + options: { op: string; name?: string; attributes?: Attributes }, + fn: () => T, + ): T; + /** Add attributes to the active span. */ + annotate(attributes: Attributes): void; + /** Add `value` to the counter `name`. */ + count(name: string, value: number, attributes: Attributes): void; + /** Add a time in milliseconds to the distribution `name`. */ + time(name: string, ms: number, attributes: Attributes): void; +} + interface Session { name: string; /** New recordings, by key. */ @@ -142,6 +166,7 @@ export interface RecorderConfig { rules: RecordingRule[]; /** Credentials to redact, in addition to the ones that it learns. */ secrets?: string[]; + telemetry: Telemetry; } /** Create the recorder of one run, with the store of its recordings. */ @@ -152,6 +177,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } } let session: Session | undefined; + const { telemetry } = config; const secrets = createSecrets(config.secrets); const stats: RecordingStats = { counts: Object.fromEntries( @@ -165,8 +191,27 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { discarded: 0, passthrough: {}, }; + /** Count recordings by rule in `roach.recording.write`. */ + const countRecordings = (keys: string[], result: "dropped" | "written") => { + const byRule = new Map(); + for (const key of keys) { + const rule = key.slice(0, key.indexOf("/")); + byRule.set(rule, (byRule.get(rule) ?? 0) + 1); + } + for (const [rule, value] of byRule) { + telemetry.count("roach.recording.write", value, { rule, result }); + } + }; const write = async (recordings: Array<[string, Recording]>) => { stats.written += await store.write(recordings); + countRecordings( + recordings.map(([key]) => key), + "written", + ); + }; + const drop = (keys: string[]) => { + stats.discarded += keys.length; + countRecordings(keys, "dropped"); }; const record = async ( @@ -177,7 +222,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { if (owner && owner.passed === undefined) { owner.recorded.set(key, recording); } else if (owner?.passed === false) { - stats.discarded += 1; + drop([key]); } else { await write([[key, recording]]); } @@ -201,7 +246,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { session = undefined; ended.passed = passed; if (passed) await write([...ended.recorded]); - else stats.discarded += ended.recorded.size; + else drop([...ended.recorded.keys()]); return ended.missed; }; @@ -216,6 +261,8 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { countPassthrough(origin: string, headers: IncomingHttpHeaders): void { secrets.learn(headers); stats.passthrough[origin] = (stats.passthrough[origin] ?? 0) + 1; + telemetry.annotate({ result: "passthrough" }); + telemetry.count("roach.request", 1, { origin, result: "passthrough" }); }, /** @@ -228,19 +275,30 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { request: ProxyRequest, sendLive: () => Promise, ): Promise { + const started = performance.now(); + const origin = request.url.origin; // A request belongs to the session that was open when it started. const owner = session; const counts = stats.counts[rule.name]!; secrets.learn(request.headers); - const keyed = keyRequest( - rule, - { - method: request.method, - url: request.url.href, - headers: request.headers, - body: request.body.toString("utf8"), + telemetry.annotate({ rule: rule.name }); + const { keyed, key } = telemetry.span( + { op: "roach.key", attributes: { rule: rule.name } }, + () => { + const result = keyRequest( + rule, + { + method: request.method, + url: request.url.href, + headers: request.headers, + body: request.body.toString("utf8"), + }, + owner?.known, + ); + const name = recordingKey(rule.name, result.key); + telemetry.annotate({ key: name }); + return { keyed: result, key: name }; }, - owner?.known, ); if (owner) { for (const [name, list] of keyed.values) { @@ -249,12 +307,22 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } } } - const key = recordingKey(rule.name, keyed.key); + const report = (result: ResponseSource) => { + const attributes = { rule: rule.name, origin, result }; + telemetry.annotate({ result }); + telemetry.count("roach.request", 1, attributes); + telemetry.time( + "roach.request.duration", + performance.now() - started, + attributes, + ); + }; if (config.mode !== "record") { const recording = await store.read(key); if (recording) { counts.replayed += 1; + report("replayed"); return { ...fromRecording(recording, keyed.values), source: "replayed", @@ -264,6 +332,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { if (config.mode === "replay") { counts.missed += 1; if (owner) owner.missed += 1; + report("missed"); return { status: 412, headers: { "content-type": "text/plain" }, @@ -276,18 +345,29 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } counts.live += 1; + const live = await telemetry.span( + { + op: "roach.upstream", + name: `${request.method} ${origin}`, + attributes: { origin, "http.request.method": request.method }, + }, + async () => { + const response = await sendLive(); + telemetry.annotate({ "http.response.status_code": response.status }); + return response; + }, + ); // The client gets the redacted response too, so a live run and a // replay give the same body. const recording = JSON.parse( secrets.redact( - JSON.stringify( - toRecording(request, keyed, owner?.name, await sendLive()), - ), + JSON.stringify(toRecording(request, keyed, owner?.name, live)), ), ) as Recording; if (!isTemporaryStatus(recording.response.status)) { await record(owner, key, recording); } + report("live"); return { ...fromRecording(recording, keyed.values), source: "live" }; }, diff --git a/src/server.ts b/src/server.ts index c46b12b..f1e172e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,7 +20,7 @@ import https from "node:https"; import { isIP, type Socket } from "node:net"; import tls from "node:tls"; import type { CertificateAuthority } from "./certificates.ts"; -import type { ProxyRequest, Recorder } from "./recorder.ts"; +import type { ProxyRequest, Recorder, Telemetry } from "./recorder.ts"; /** The path prefix of the control API. */ export const CONTROL_PATH = "/__roach"; @@ -289,6 +289,8 @@ export interface ProxyTarget { replayOnly?: boolean; /** Set when the target stops. Its tunnels then refuse new requests. */ closed?: boolean; + /** Gets a span for each proxied request. */ + telemetry: Telemetry; } /** A target, or the HTTP status that refuses the request. */ @@ -345,8 +347,24 @@ function serveProxied( outgoing.end("Roach: this run has ended\n"); return; } - proxyRequest(target, incoming, outgoing, tunnelOrigin).catch( - (error: unknown) => { + // Only the origin: the path and query can hold a credential. + const origin = tunnelOrigin ?? URL.parse(incoming.url ?? "")?.origin; + const method = incoming.method ?? "GET"; + target.telemetry + .span( + { + op: "roach.request", + name: `${method} ${origin}`, + attributes: { origin: String(origin), "http.request.method": method }, + }, + async () => { + await proxyRequest(target, incoming, outgoing, tunnelOrigin); + target.telemetry.annotate({ + "http.response.status_code": outgoing.statusCode, + }); + }, + ) + .catch((error: unknown) => { if (!outgoing.headersSent) { const status = (error as { status?: number }).status ?? 502; outgoing.writeHead(status, { "content-type": "text/plain" }); @@ -354,8 +372,7 @@ function serveProxied( outgoing.end( `Roach error: ${error instanceof Error ? error.message : String(error)}\n`, ); - }, - ); + }); } /** diff --git a/src/service.ts b/src/service.ts index 415289d..13bbbaa 100644 --- a/src/service.ts +++ b/src/service.ts @@ -19,8 +19,9 @@ * - The recordings of a tenant are under `/`, in a directory or a * GCS bucket (`gcs.ts`). Tenants never share recordings, because a key * does not include credentials. - * - Each read and write of a recording sends the Sentry metric - * `roach.recording`, with the tenant, the key, and the run. + * - With `sentryDsn`, the service sends spans and metrics to Sentry. See + * "Sentry" in `README.md`. Metrics have only attributes with few values, + * so the key and the run are only on spans. */ import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; import type http from "node:http"; @@ -31,7 +32,7 @@ import { type CertificateAuthorityPem, } from "./certificates.ts"; import { createGcsStore } from "./gcs.ts"; -import { createRecorder } from "./recorder.ts"; +import { createRecorder, type Telemetry } from "./recorder.ts"; import { createFileStore } from "./recordings.ts"; import { CONTROL_PATH, @@ -141,6 +142,8 @@ interface Run extends ProxyTarget { } const MODES = new Set(["auto", "off", "record", "replay"]); +/** The service gets little traffic, so it keeps all traces. */ +const TRACES_SAMPLE_RATE = 1; /** A run that is open longer than this ends as failed, such as a dead job. */ const RUN_TIMEOUT_MS = 6 * 60 * 60 * 1000; /** @@ -249,36 +252,73 @@ function invalidRunConfig( } /** - * The store of one run. It counts each read and write in Sentry, and a - * run without write access cannot write. + * The telemetry of one run, in Sentry. Metrics get `labels`. Spans also get + * the name of the run. + */ +function runTelemetry( + labels: { tenant: string; mode: RecordingMode }, + run: string, +): Telemetry { + return { + span: ({ op, name = op, attributes }, fn) => + Sentry.startSpan( + { op, name, attributes: { ...labels, run, ...attributes } }, + fn, + ), + annotate: (attributes) => { + Sentry.getActiveSpan()?.setAttributes(attributes); + }, + count: (name, value, attributes) => { + Sentry.metrics.count(name, value, { + attributes: { ...labels, ...attributes }, + }); + }, + time: (name, ms, attributes) => { + Sentry.metrics.distribution(name, ms, { + unit: "millisecond", + attributes: { ...labels, ...attributes }, + }); + }, + }; +} + +/** + * The store of one run. Each read and write has a span, and a run without + * write access cannot write. */ function runStore( base: RecordingStore, - labels: { tenant: string; run: string }, + labels: { tenant: string; mode: RecordingMode; run: string }, canWrite: boolean, ): RecordingStore { - const count = (key: string, result: string) => { - Sentry.metrics.count("roach.recording", 1, { - attributes: { - ...labels, - rule: key.slice(0, key.indexOf("/")), - key, - result, - }, - }); - }; + const span = ( + op: string, + attributes: Record, + fn: (span: Sentry.Span) => Promise, + ) => + Sentry.startSpan( + { op, name: op, attributes: { ...labels, ...attributes } }, + fn, + ); return { - async read(key) { - const recording = await base.read(key); - count(key, recording ? "replayed" : "missed"); - return recording; - }, + read: (key) => + span("roach.store.read", { key }, async (current) => { + const recording = await base.read(key); + current.setAttribute("result", recording ? "hit" : "miss"); + return recording; + }), async write(recordings) { if (recordings.length === 0) return 0; if (!canWrite) throw new Error("This run cannot write recordings"); - const written = await base.write(recordings); - for (const [key] of recordings) count(key, "written"); - return written; + return span( + "roach.store.write", + { recordings: recordings.length }, + async (current) => { + const written = await base.write(recordings); + current.setAttribute("written", written); + return written; + }, + ); }, }; } @@ -295,7 +335,26 @@ export async function startRoachService( } const writeTokenHash = Buffer.from(config.writeTokenHash, "hex"); if (config.sentryDsn) { - Sentry.init({ dsn: config.sentryDsn, tracesSampleRate: 0 }); + Sentry.init({ + dsn: config.sentryDsn, + tracesSampleRate: TRACES_SAMPLE_RATE, + // Send only what Roach puts on its spans. The HTTP integrations and + // `RequestData` add whole URLs, headers, and bodies, which can hold + // credentials, such as the `secrets` of a run. And the proxy must not + // add trace headers to the requests that it sends. + tracePropagationTargets: [], + integrations: (defaults) => [ + ...defaults.filter( + ({ name }) => !["Http", "NodeFetch", "RequestData"].includes(name), + ), + Sentry.httpIntegration({ + spans: false, + breadcrumbs: false, + maxRequestBodySize: "none", + }), + Sentry.nativeNodeFetchIntegration({ spans: false, breadcrumbs: false }), + ], + }); } const serviceOrigins = parseOrigins(config.allow); const runs = new Map(); @@ -319,11 +378,19 @@ export async function startRoachService( const createRun = (runConfig: RunConfig, canWrite: boolean): RemoteRun => { const id = randomBytes(8).toString("hex"); + const telemetry = runTelemetry( + { tenant: runConfig.tenant, mode: runConfig.mode }, + runConfig.name ?? id, + ); const token = randomBytes(24).toString("hex"); const allow = runConfig.allow ?? config.allow; const store = runStore( storeOf(runConfig.tenant), - { tenant: runConfig.tenant, run: runConfig.name ?? id }, + { + tenant: runConfig.tenant, + mode: runConfig.mode, + run: runConfig.name ?? id, + }, canWrite, ); const run: Run = { @@ -339,9 +406,11 @@ export async function startRoachService( mode: runConfig.mode, rules: runConfig.rules, secrets: [...(config.secrets ?? []), ...(runConfig.secrets ?? [])], + telemetry, }, store, ), + telemetry, }; runs.set(id, run); process.stderr.write( @@ -415,7 +484,20 @@ export async function startRoachService( return; } } - sendJson(outgoing, 201, createRun(runConfig as RunConfig, canWrite)); + const valid = runConfig as RunConfig; + const remote = Sentry.startSpan( + { + op: "roach.run.start", + name: "roach.run.start", + attributes: { tenant, mode: valid.mode, write: canWrite }, + }, + (span) => { + const created = createRun(valid, canWrite); + span.setAttribute("run", valid.name ?? created.id); + return created; + }, + ); + sendJson(outgoing, 201, remote); return; } diff --git a/tests/deployed.test.ts b/tests/deployed.test.ts index 1ba5b1a..0b36bf7 100644 --- a/tests/deployed.test.ts +++ b/tests/deployed.test.ts @@ -14,7 +14,7 @@ import { execFile, spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; import { once } from "node:events"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { createServer } from "node:http"; +import { createServer, type IncomingHttpHeaders } from "node:http"; import https from "node:https"; import net, { type AddressInfo } from "node:net"; import { tmpdir } from "node:os"; @@ -35,7 +35,16 @@ let directory: string; let hostCertFile: string; let caCert: string; const objects = new Map(); -const metrics: Array> = []; +const envelopes: string[] = []; +/** A metric or a span that the service sent to Sentry. */ +interface TelemetryItem { + name: string; + value?: number; + attributes: Record; +} +const metrics: TelemetryItem[] = []; +const spans: TelemetryItem[] = []; +const upstreamHeaders: IncomingHttpHeaders[] = []; let liveRequests = 0; let upstreamOrigin: string; let frontUrl: string; @@ -59,23 +68,58 @@ async function listen(server: net.Server, host = "127.0.0.1") { return (server.address() as AddressInfo).port; } -/** Keep the `roach.recording` metrics of a Sentry envelope. */ +/** + * Keep the metrics and the spans of a Sentry envelope. Both are lists of + * items with typed attributes. + */ function readEnvelope(envelope: string) { - for (const line of envelope.split("\n")) { - if (!line.includes('"roach.recording"')) continue; - const { items } = JSON.parse(line) as { - items: Array<{ attributes: Record }>; + envelopes.push(envelope); + // A header line, then a header line and a payload line for each item. + const [, ...lines] = envelope.split("\n").filter(Boolean); + for (let index = 0; index + 1 < lines.length; index += 2) { + const { type } = JSON.parse(lines[index]!) as { type: string }; + const list = { trace_metric: metrics, span: spans }[type]; + if (!list) continue; + const { items } = JSON.parse(lines[index + 1]!) as { + items: Array< + Omit & { + attributes: Record; + } + >; }; - for (const { attributes } of items) { - metrics.push( - Object.fromEntries( - Object.entries(attributes).map(([name, { value }]) => [name, value]), + for (const item of items) { + list.push({ + ...item, + attributes: Object.fromEntries( + Object.entries(item.attributes).map(([name, { value }]) => [ + name, + value, + ]), ), - ); + }); } } } +/** The metrics of one request of the `model` rule. */ +function requestMetrics(mode: string, result: string) { + const attributes = { + tenant: TENANT, + mode, + rule: "model", + origin: upstreamOrigin, + result, + }; + return [ + { name: "roach.request", value: 1, attributes }, + { + name: "roach.request.duration", + value: expect.any(Number), + attributes, + }, + ]; +} + /** * Run one CI job: the action (`src/action.ts`) with a command that sends * one HTTPS request with plain `fetch`. Returns the exit code of the @@ -197,8 +241,9 @@ beforeAll(async () => { }; // The upstream API. It is not on `localhost`, so `NO_PROXY` keeps it. - const upstream = https.createServer(host, (_incoming, outgoing) => { + const upstream = https.createServer(host, (incoming, outgoing) => { liveRequests += 1; + upstreamHeaders.push(incoming.headers); outgoing.writeHead(200, { "content-type": "application/json" }); outgoing.end(JSON.stringify({ n: liveRequests })); }); @@ -361,23 +406,106 @@ describe("deployed roach", () => { expect(await once(service, "exit")).toEqual([0, null]); const [key] = objects.keys(); const relativeKey = key!.slice(`${TENANT}/`.length); + + // Metrics have only attributes with few values: no key and no run. + const labels = ["tenant", "mode", "rule", "origin", "result", "key", "run"]; expect( - metrics.map(({ tenant, run, result, key: metricKey }) => ({ - tenant, - run, - result, - key: metricKey, + metrics.map(({ name, value, attributes }) => ({ + name, + value, + attributes: Object.fromEntries( + labels.flatMap((label) => + label in attributes ? [[label, attributes[label]]] : [], + ), + ), })), ).toEqual([ - { tenant: TENANT, run: "writer-1", result: "missed", key: relativeKey }, - { tenant: TENANT, run: "writer-1", result: "written", key: relativeKey }, - { tenant: TENANT, run: "fork-1", result: "replayed", key: relativeKey }, + ...requestMetrics("auto", "live"), { - tenant: TENANT, - run: "miss-1", - result: "missed", - key: expect.stringMatching(/^model\//), + name: "roach.recording.write", + value: 1, + attributes: { + tenant: TENANT, + mode: "auto", + rule: "model", + result: "written", + }, }, + ...requestMetrics("replay", "replayed"), + ...requestMetrics("replay", "missed"), ]); + + // Only the spans of Roach: 4 for each run, and 2 for the live request. + const ops = spans.map(({ attributes }) => attributes["sentry.op"]); + expect(ops).toHaveLength(14); + expect(new Set(ops)).toEqual( + new Set([ + "roach.run.start", + "roach.request", + "roach.key", + "roach.store.read", + "roach.upstream", + "roach.store.write", + ]), + ); + const span = (op: string, run: string) => + spans.find( + ({ attributes }) => + attributes["sentry.op"] === op && attributes.run === run, + )?.attributes; + expect(span("roach.run.start", "writer-1")).toMatchObject({ + tenant: TENANT, + mode: "auto", + write: true, + }); + expect(span("roach.request", "writer-1")).toMatchObject({ + tenant: TENANT, + mode: "auto", + rule: "model", + origin: upstreamOrigin, + result: "live", + "http.response.status_code": 200, + }); + expect(span("roach.request", "fork-1")).toMatchObject({ + result: "replayed", + }); + expect(span("roach.request", "miss-1")).toMatchObject({ + result: "missed", + "http.response.status_code": 412, + }); + expect(span("roach.key", "writer-1")).toMatchObject({ key: relativeKey }); + expect(span("roach.store.read", "writer-1")).toMatchObject({ + key: relativeKey, + result: "miss", + }); + expect(span("roach.store.read", "fork-1")).toMatchObject({ + key: relativeKey, + result: "hit", + }); + expect(span("roach.upstream", "writer-1")).toMatchObject({ + origin: upstreamOrigin, + "http.response.status_code": 200, + }); + expect(span("roach.store.write", "writer-1")).toMatchObject({ + recordings: 1, + written: 1, + }); + + // Sentry gets no credential, no path, and no body, and the upstream + // gets no trace headers. + for (const secret of [ + TOKEN, + GCS_TOKEN, + "/v1/messages", + "/__roach/", + "prompt", + ]) { + expect(envelopes.join("\n")).not.toContain(secret); + } + for (const headers of upstreamHeaders) { + expect(headers).not.toHaveProperty("sentry-trace"); + expect(headers).not.toHaveProperty("baggage"); + expect(headers).not.toHaveProperty("traceparent"); + } }); });