diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 5c536c6..929d14d 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -450,6 +450,25 @@ jobs: publish-results: needs: [prepare, run-evals] runs-on: ubuntu-latest + # A way to say "run, but do not publish this week". + # + # Publishing was gated on the matrix succeeding and nothing else, so a run + # against a `main` we already knew was wrong would publish anyway. On 24 + # August this fired at 06:14 with the addendum defect still on `main` — the + # one that made twelve baseline cells fail for a reason that was ours — and + # was cancelled by hand at roughly 47 of 72 cells, minutes ahead of this + # job. The fixes existed, reviewed, on an open PR. The cron could neither + # know that nor be told. + # + # Cancelling by hand needs somebody watching at the right moment, and nobody + # was; it came up in conversation by luck. Set the repository variable + # `EVALS_PUBLISH` to `false` and the matrix still runs and still uploads its + # artifacts — the evidence is kept — while `results/` is left alone. + # + # Deliberately a variable rather than a secret: it is configuration, it + # should be visible in the run log, and anyone reading a skipped publish + # should be able to see why without repository admin. + if: vars.EVALS_PUBLISH != 'false' steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 diff --git a/.gitignore b/.gitignore index 21f0052..82c9215 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,12 @@ dist/ .hookdeck-pristine.json **/.hookdeck-pristine.json +# The Outpost deployment's pristine managed config, recorded on first acquire so +# a crashed run cannot make its damage permanent (#41). Machine-local, like the +# project snapshot above. +.outpost-pristine-config.json +**/.outpost-pristine-config.json + # Agent worktrees. Real git repos; never part of this one. .claude/worktrees/ diff --git a/AGENTS.md b/AGENTS.md index 61f3170..44f3dae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -329,6 +329,27 @@ release are not comparable, and the notes have to say so rather than showing a d that reads as improvement. The same applies to a sandbox CLI bump, which changes the product under test. +**A run can be told not to publish.** Set the repository variable +`EVALS_PUBLISH` to `false` and the weekly matrix still runs and still uploads +its artifacts, while `results/` is left alone: + +```bash +gh variable set EVALS_PUBLISH --body false # hold publishing +gh variable delete EVALS_PUBLISH # resume +``` + +Use it whenever `main` carries something you already know is wrong — a scorer +mid-repair, a scenario whose seed is being rewritten, a harness fix reviewed but +unmerged. On 24 August the cron fired against a `main` whose prompt addendum was +still omitting a credential, and was cancelled by hand at 47 of 72 cells, +minutes ahead of publishing twelve failures that were ours rather than the +agents'. That required somebody to be watching, and it was luck that anybody +was. + +**A held run is still worth running.** The artifacts carry the transcripts, and +transcripts are where the findings come from — the scoreboard has never produced +one. Holding publication is not the same as skipping the week. + ## Plans `.plans/` holds the planning documents. Start with diff --git a/packages/hookdeck/src/project-source.ts b/packages/hookdeck/src/project-source.ts index 0d172f8..eb072b0 100644 --- a/packages/hookdeck/src/project-source.ts +++ b/packages/hookdeck/src/project-source.ts @@ -65,6 +65,8 @@ export interface FixedProjectSourceOptions { projectId?: string; /** Where the pristine snapshot is persisted between runs. */ snapshotPath?: string; + /** Where the pristine Outpost managed config is recorded. See #41. */ + outpostConfigPath?: string; baseUrl?: string; /** * Set when scenarios requiring Outpost can run. Outpost is a separate @@ -80,6 +82,7 @@ export class FixedProjectSource implements ProjectSource { private readonly options: FixedProjectSourceOptions; private readonly projectId: string; private readonly snapshotPath: string; + private readonly outpostConfigPath: string; private cachedClient?: HookdeckClient; private cachedOutpostClient?: OutpostClient; private outpostTenantsAtAcquire?: Set; @@ -90,6 +93,8 @@ export class FixedProjectSource implements ProjectSource { this.options = options; this.projectId = options.projectId ?? 'evals-ci'; this.snapshotPath = options.snapshotPath ?? '.hookdeck-pristine.json'; + this.outpostConfigPath = + options.outpostConfigPath ?? '.outpost-pristine-config.json'; } /** @@ -142,7 +147,7 @@ export class FixedProjectSource implements ProjectSource { const snapshot = await this.loadOrCaptureSnapshot(); await this.resetToPristine(snapshot); this.outpostTenantsAtAcquire = await this.listOutpostTenants(); - this.outpostConfigAtAcquire = await this.readOutpostConfig(); + this.outpostConfigAtAcquire = await this.loadOrCaptureOutpostConfig(); this.operatorEventDestinationsAtAcquire = await this.readOperatorEventDestinations(); return { @@ -211,6 +216,56 @@ export class FixedProjectSource implements ProjectSource { } } + /** + * The config a pristine project has, read from disk when we have recorded it. + * + * In-memory capture was not enough, and the failure was not hypothetical: on + * 21 August an agent trying to configure operator events sent 21 `PATCH + * /config` requests and left `TOPICS` empty. The run died before release, the + * *next* acquire captured the broken value as its baseline, and every seed + * afterwards failed with `422 invalid topics` — nineteen of twenty-four cells, + * and the original value was unrecoverable because it had already been + * overwritten before anyone read it. + * + * Deployment config is worse than a leaked tenant in two ways. It is global, + * so it breaks scenarios that never touch config; and it has no natural + * baseline, so "pristine" silently becomes whatever the last crash left. + * + * On disk it survives a crash, exactly as `.hookdeck-pristine.json` does for + * the project itself. Values that come back redacted are not recorded, so a + * mask can never be restored as though it were the setting. + */ + private async loadOrCaptureOutpostConfig(): Promise< + Record | undefined + > { + if (!this.outpostClient) return undefined; + + try { + return JSON.parse(readFileSync(this.outpostConfigPath, 'utf8')) as Record< + string, + unknown + >; + } catch { + // No baseline recorded yet: today's values are the best available one. + const live = await this.readOutpostConfig(); + if (!live) return undefined; + const recordable = Object.fromEntries( + Object.entries(live).filter(([, v]) => !looksRedacted(v)) + ); + try { + mkdirSync(dirname(this.outpostConfigPath), { recursive: true }); + writeFileSync( + this.outpostConfigPath, + JSON.stringify(recordable, null, 2) + ); + } catch { + // Recording is best effort; restoring from memory still works for a + // run that exits cleanly. + } + return live; + } + } + /** * Put back any managed config value this lease changed. *