feat(service): Redact the secrets that a run sends

README.md

@@ -102,7 +102,7 @@ the run and writes its stats to the job summary.
 ## Config
 
 A run takes `RunConfig` (`src/service.ts`): `tenant`, `mode`, `rules`,
-`allow`, and `name`.
+`allow`, `name`, and `secrets`.
 
 - `mode`: `auto` replays recordings and records misses. `replay` replays
   recordings and fails a miss with HTTP 412, so nothing goes live. `record`
@@ -115,8 +115,11 @@ A run takes `RunConfig` (`src/service.ts`): `tenant`, `mode`, `rules`,
   and `-`. `match` has `method`, `url` (a prefix),
   and `headers`. `keyHeaders` adds request headers to the key. `values`
   names the values that change from run to run.
-- Credentials: the proxy never writes the `secrets` of the service config.
-  It also learns the value of each request header whose name can mean a credential,
+- Credentials: the proxy never writes the `secrets` of the service config
+  or of the run. Use `secrets` of the run for the credentials of a CI job
+  that a response can show but no request header has, such as a token that
+  goes to a model in a prompt. The service keeps them only in memory, for
+  the run. A run can have 32 of them. It also learns the value of each request header whose name can mean a credential,
   such as one with `auth`, `token`, `key`, `secret`, `cookie`, or `session`.
   The match is broad on purpose. Recordings show `<<redacted>>` in place of
   each known value.

src/secrets.ts

@@ -24,7 +24,11 @@ export function createSecrets(initial: string[] = []) {
   // `Bearer <token>` or each value of a cookie header.
   const add = (value: string) => {
     for (const part of [value, ...value.split(/[\s;,=]+/)]) {
-      if (part.length >= MIN_LENGTH) values.add(part);
+      if (part.length < MIN_LENGTH) continue;
+      values.add(part);
+      // The recorder redacts JSON text, which escapes `"`, `\`, and
+      // control characters.
+      values.add(JSON.stringify(part).slice(1, -1));
     }
   };
   initial.forEach(add);

src/service.ts

@@ -104,6 +104,12 @@ export interface RunConfig {
    * `${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}`. Default: the run id.
    */
   name?: string;
+  /**
+   * Credentials that this run must never write, such as the API keys of
+   * its CI job. The service redacts them with the `secrets` of the service
+   * config, keeps them only in memory, and forgets them when the run ends.
+   */
+  secrets?: string[];
 }
 
 /** A run that the service created. */
@@ -144,6 +150,9 @@ const RUN_TIMEOUT_MS = 6 * 60 * 60 * 1000;
  */
 const MAX_PUBLIC_RUNS = 100;
 const MAX_RUN_NAME = 128;
+/** Enough for the API keys and OIDC tokens of a CI job. */
+const MAX_RUN_SECRETS = 32;
+const MAX_SECRET_LENGTH = 16 * 1024;
 /**
  * A tenant name: `owner/repo`, with the characters of GitHub names. It is a
  * path in the store, so each part starts with a letter, digit, or `_`, and
@@ -188,6 +197,18 @@ function invalidRunConfig(
       return `name must be a string of at most ${MAX_RUN_NAME} characters`;
     }
   }
+  if (config.secrets !== undefined) {
+    if (
+      !Array.isArray(config.secrets) ||
+      config.secrets.length > MAX_RUN_SECRETS ||
+      !config.secrets.every(
+        (secret) =>
+          typeof secret === "string" && secret.length <= MAX_SECRET_LENGTH,
+      )
+    ) {
+      return `secrets must be a list of at most ${MAX_RUN_SECRETS} strings of at most ${MAX_SECRET_LENGTH} characters`;
+    }
+  }
   if (!Array.isArray(config.rules)) return "rules must be a list";
   const names = new Set<string>();
   for (const rule of config.rules) {
@@ -317,7 +338,7 @@ export async function startRoachService(
         {
           mode: runConfig.mode,
           rules: runConfig.rules,
-          ...(config.secrets ? { secrets: config.secrets } : {}),
+          secrets: [...(config.secrets ?? []), ...(runConfig.secrets ?? [])],
         },
         store,
       ),

tests/service.test.ts

@@ -89,6 +89,7 @@ beforeEach(async () => {
       JSON.stringify({
         n: liveRequests,
         key: incoming.headers["x-api-key"] ?? null,
+        echo: incoming.headers["x-echo"] ?? null,
         serviceSecret: "service-secret-value-123",
       }),
     );
@@ -233,6 +234,9 @@ describe("roach service", () => {
         rules: [{ name: "model", match: {}, values: { slow: "(a+)+$" } }],
       }),
     ).rejects.toThrow(/not a pattern that the service allows/);
+    await expect(
+      startRun("write-token", "auto", { secrets: [42 as unknown as string] }),
+    ).rejects.toThrow(/secrets must be a list/);
     await startRun("write-token", "auto", {
       rules: [
         { name: "model", match: {}, values: { id: VALUE_PATTERNS.uuid } },
@@ -281,18 +285,26 @@ describe("roach service", () => {
     expect(liveRequests).toBe(1);
   });
 
-  it("redacts request credentials and the secrets of the service", async () => {
-    const run = await startRun("write-token", "auto");
+  it("redacts request credentials and the secrets of the service and the run", async () => {
+    // A run secret is not in a credential header, so only the run config
+    // tells the service about it.
+    const run = await startRun("write-token", "auto", {
+      secrets: ["run-secret-value-789"],
+    });
     const response = await send(
       run.url,
       { prompt: "hi" },
-      { "x-api-key": "sk-live-request-key-456" },
+      {
+        "x-api-key": "sk-live-request-key-456",
+        "x-echo": "run-secret-value-789",
+      },
     );
 
     // The client gets the redacted body, as a replay would give it.
     expect(JSON.parse(response.body)).toEqual({
       n: 1,
       key: "<<redacted>>",
+      echo: "<<redacted>>",
       serviceSecret: "<<redacted>>",
     });
     const [file] = await recordings("acme/alpha");
@@ -302,5 +314,6 @@ describe("roach service", () => {
     );
     expect(text).not.toContain("sk-live-request-key-456");
     expect(text).not.toContain("service-secret-value-123");
+    expect(text).not.toContain("run-secret-value-789");
   });
 });