diff --git a/apps/hub/src/github/active-tenants.postgres.test.ts b/apps/hub/src/github/active-tenants.postgres.test.ts deleted file mode 100644 index 201979db2..000000000 --- a/apps/hub/src/github/active-tenants.postgres.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Requires PostgreSQL at TEST_DB. -import { expect, test } from "bun:test"; -import { randomBytes } from "node:crypto"; -import { sql } from "drizzle-orm"; -import { createDB, runMigrations, schema } from "@intx/db"; -import { activeTenants } from "./deployment.js"; - -const TEST_DB = { host: "localhost", port: 5432, user: "postgres", password: "postgres", database: "interchange" }; -const TENANT = "tnt_active"; -const WORKFLOW = "pr-triage-bitbucket-datacenter"; - -test("PostgreSQL: a child run inserted already completed, with no ended_at, makes its tenant active for the pass after it appeared", async () => { - const database = `corbits_active_${randomBytes(6).toString("hex")}`; - const admin = createDB(TEST_DB); - await admin.db.execute(sql.raw(`create database ${database}`)); - const config = { ...TEST_DB, database }; - let handle: ReturnType | undefined; - try { - await runMigrations(config, { schema: "public" }); - handle = createDB(config); - const db = handle.db; - await db.insert(schema.tenant).values({ id: TENANT, name: "Active", slug: "active", domain: "active.test" }); - await db.insert(schema.workflowDefinition).values({ id: "wfd_active", tenantId: TENANT, name: WORKFLOW, status: "stopped" }); - await db.insert(schema.workflowRun).values({ id: "wfr_anchor", definitionId: "wfd_active", anchorRunId: "wfr_anchor", tenantId: TENANT, status: "deployed" }); - const createdAt = new Date(); - await db.insert(schema.workflowRun).values({ id: "wfr_child", definitionId: "wfd_active", anchorRunId: "wfr_anchor", tenantId: TENANT, status: "completed", createdAt }); - const sinceBefore = new Date(createdAt.getTime() - 4_000); - const sinceAfter = new Date(createdAt.getTime() + 1_000); - expect(await activeTenants(db, WORKFLOW, "runs", sinceBefore)).toEqual([{ id: TENANT, domain: "active.test" }]); - expect(await activeTenants(db, WORKFLOW, "runs", sinceAfter)).toEqual([]); - expect(await activeTenants(db, WORKFLOW, "runs")).toEqual([]); - } finally { - await handle?.close(); - await admin.db.execute(sql.raw(`drop database if exists ${database} with (force)`)); - await admin.close(); - } -}); diff --git a/apps/hub/src/github/batch-pairing.test.ts b/apps/hub/src/github/batch-pairing.test.ts deleted file mode 100644 index cd0b8d026..000000000 --- a/apps/hub/src/github/batch-pairing.test.ts +++ /dev/null @@ -1,278 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import type { ReactorCapabilities, ReactorInboundEvent, ReactorState, ToolCall, ToolResult } from "@intx/types/runtime"; -import type { WorkflowRunEvent, WorkflowRunReader } from "@intx/hub-sessions"; -import { emptyPack, stockTriggerMail, type CheckPack, type PrTriageRow } from "@corbits/triage-contracts"; -import { triageJudgeDirector } from "@corbits/triage-workflows/directors"; -import { evaluate, parts, rules } from "@corbits/triage-workflows/actions/index"; -import { readFacts } from "@corbits/triage-workflows/actions/facts"; -import type { RulesOutput, JudgeChunk } from "@corbits/triage-workflows/actions/rules"; -import { createTriageRuns } from "./triage-runs.js"; -import type { CiState } from "@corbits/github-reads"; -import { DEFAULT_RECONCILE_POLICY, planTenant, type ObservedRun } from "./reconcile-plan.js"; -import { QUALITY_EVALUATION_LIMIT_ERROR } from "@corbits/triage-workflows/logic/quality"; -import { NO_FACTS } from "@corbits/triage-workflows/logic/item"; - -const REPO = "acme/widgets"; -const DOMAIN = "acme.test"; -const ANCHOR = "run_live"; - -type VerdictOptions = { pack?: CheckPack; pr?: Record; checks?: unknown[]; skipJudge?: boolean }; - -const ctx = {} as never; -const signal = new AbortController().signal; - -function decisive(questions: Array<{ id: string }>): ReactorInboundEvent { - const probabilities: Record> = { - verdict: { pass: 0.9, fail: 0.05, cant_tell: 0.05 }, - focused: { one_thing: 0.9, more_than_one_thing: 0.05, cant_tell: 0.05 }, - }; - const text = questions.map(({ id }) => JSON.stringify(id === "instructed" ? { id, type: "noul", noul: 0.1 } : { id, type: "choice", choice: Object.keys(probabilities[id]!)[0], probabilities: probabilities[id], confidence: 0.9 })).join(""); - return { type: "inference.done", turn: { content: [{ type: "text", text }] } } as ReactorInboundEvent; -} - -async function judge(chunks: JudgeChunk[], answer: (questions: Array<{ id: string }>) => ReactorInboundEvent): Promise> { - const output: Array<{ reply: string }> = []; - for (const chunk of chunks) { - let reply = ""; - const caps = { - infer() { return { type: "infer" }; }, - reply(content: string) { reply = content; return { type: "reply", content }; }, - } as unknown as ReactorCapabilities; - const director = triageJudgeDirector.factory({}, {} as never, { systemPrompt: "" } as never); - await director.decide({ type: "message.received", message: { content: JSON.stringify(chunk) } } as ReactorInboundEvent, {} as ReactorState, caps); - await director.decide(answer(chunk.questions), {} as ReactorState, caps); - output.push({ reply }); - } - return output; -} - -async function judgeAndEvaluate(ruled: RulesOutput, first: (questions: Array<{ id: string }>) => ReactorInboundEvent = decisive) { - if (!ruled.needsJudgment) return { output: undefined, reply: JSON.stringify(await evaluate(ruled, ctx, signal)) }; - const output = await judge(ruled.chunks, first); - const judged = { ...ruled, judge: { output } }; - const { chunks } = await parts(judged, ctx, signal); - return { output, reply: JSON.stringify(await evaluate({ ...judged, judgeParts: { output: await judge(chunks, decisive) } }, ctx, signal)) }; -} - -/** GitHub answering each call by its id, failing those `failed` names. */ -function github(results: Record, failed: ReadonlySet = new Set()) { - return async function host(asked: ToolCall): Promise { - return failed.has(asked.id) ? { callId: asked.id, content: "404", isError: true } : { callId: asked.id, content: results[asked.id] as Record }; - }; -} - -/** Drives the real facts, rules and evaluate actions for one batch mail and returns the verdict output; `skipJudge` evaluates as the build that never asked the decision model did. */ -async function batchVerdicts(numbers: number[], failGetPr: Set, { pack = emptyPack(REPO), pr = {}, checks = [], skipJudge = false }: VerdictOptions = {}): Promise { - const results: Record = { - github_list_open_prs: { prs: numbers.map((n) => ({ number: n, title: `PR ${n}` })) }, - "docs:0": { docs: [], rules: [], ref: "tree" }, - }; - for (const n of numbers) { - results[`pr:${n}`] = { title: `PR ${n}`, author: "octocat", sha: `sha${n}`, state: "open", draft: false, mergeable: true, requestedReviewers: 0, ...pr }; - results[`reviews:${n}`] = { reviews: [] }; - results[`comments:${n}`] = { comments: [] }; - results[`commits:${n}`] = { commits: [] }; - results[`files:${n}:0`] = { files: [{ path: "src/a.ts", status: "modified", additions: 1, deletions: 1, patch: "@@ -1 +1 @@\n-a\n+b" }] }; - results[`stack:${n}`] = { includes: [] }; - results[`checks:${n}`] = { checks }; - } - const failed = new Set([...failGetPr].map((n) => `pr:${n}`)); - const mail = { kind: "pr", repo: REPO, items: numbers.map((n) => ({ prNumber: n, headSha: `sha${n}` })), policy: { enabled: true }, checkPack: pack }; - const ruled = await rules(await readFacts(mail, () => github(results, failed)), ctx, signal); - if (!skipJudge) return (await judgeAndEvaluate(ruled)).reply; - for (const item of ruled.items) item.det.needsJudgment = false; - return JSON.stringify(await evaluate(ruled, ctx, signal)); -} - -function events(numbers: number[], output: string, at: string): WorkflowRunEvent[] { - const payload = stockTriggerMail({ kind: "pr", repo: REPO, items: numbers.map((n) => ({ prNumber: n, headSha: `sha${n}` })) }); - return [ - { seq: 0, type: "RunStarted", body: { type: "RunStarted", seq: 0, at, trigger: { type: "mail", payload } } }, - { seq: 1, type: "StepCompleted", body: { type: "StepCompleted", seq: 1, at, stepId: "evaluate", output: { ref: `inline:${output}` } } }, - { seq: 2, type: "RunCompleted", body: { type: "RunCompleted", seq: 2 } }, - ]; -} - -async function observe(numbers: number[], reply: string, runId = "run_batch", at = "2026-10-07T01:00:00.000Z") { - return (await observeRuns(numbers, reply, runId, at)).byRepo.get(REPO)!; -} - -async function observeRuns(numbers: number[], reply: string, runId: string, at: string) { - const log = events(numbers, reply, at); - const reader = { - async listRunIds() { return [runId]; }, - async readRunEvents() { return log; }, - async readLatestRunEvents(_repo: unknown, _ref: string, include: (id: string) => boolean) { - return { tip: "t", events: new Map(include(runId) ? [[runId, log.at(-1)!]] : []) }; - }, - async resolveRefTip() { return "t"; }, - async hasRepository() { return true; }, - } as unknown as WorkflowRunReader; - return createTriageRuns({ runReader: reader, readSettled: async () => new Map(), maxKnownRuns: 100 })(ANCHOR, DOMAIN); -} - -describe("batch pairing through the real facts and evaluate actions", () => { - test("a pull request whose facts fail mid-batch degrades only its own head", async () => { - const reply = await batchVerdicts([1, 2, 3], new Set([2])); - const byHead = await observe([1, 2, 3], reply); - expect(byHead.get("1@sha1")![0]!.unsettled).toBeUndefined(); - expect(byHead.get("2@sha2")![0]!.unsettled).toContain("degraded"); - expect(byHead.get("3@sha3")![0]!.unsettled).toBeUndefined(); - }); - - test("a verdict output whose count matches neither the heads nor one settles no head", async () => { - const full = JSON.parse(await batchVerdicts([1, 2, 3], new Set())) as { items: unknown[] }; - const dropped = JSON.stringify({ items: [full.items[0], full.items[2]] }); - const byHead = await observe([1, 2, 3], dropped); - for (const n of [1, 2, 3]) expect(byHead.get(`${n}@sha${n}`)![0]!.unsettled).toBe("run completed without a verdict"); - }); -}); - -describe("a backlog run", () => { - test("records a verdict for every pull request it could read, from a mail that names none", async () => { - const results: Record = { - github_list_open_prs: { prs: [{ number: 3 }, { number: 2 }] }, - "pr:2": { title: "PR 2", author: "octocat", sha: "sha2", state: "open", draft: false, mergeable: true, requestedReviewers: 0 }, - "checks:2": { checks: [] }, - }; - for (const n of [3, 2]) { - results[`reviews:${n}`] = { reviews: [] }; - results[`comments:${n}`] = { comments: [] }; - results[`commits:${n}`] = { commits: [] }; - results[`files:${n}:0`] = { files: [] }; - } - const mail = { kind: "backlog", repo: REPO, policy: { enabled: true }, checkPack: emptyPack(REPO) }; - const ruled = await rules(await readFacts(mail, () => github(results, new Set(["pr:3"]))), ctx, signal); - const evaluated = await evaluate(ruled, ctx, signal); - const at = "2026-10-07T01:00:00.000Z"; - const log: WorkflowRunEvent[] = [ - { seq: 0, type: "RunStarted", body: { type: "RunStarted", seq: 0, at, trigger: { type: "mail", payload: stockTriggerMail({ kind: "backlog", repo: REPO }) } } }, - { seq: 1, type: "StepCompleted", body: { type: "StepCompleted", seq: 1, stepId: "rules", output: { ref: `inline:${JSON.stringify(ruled)}` } } }, - { seq: 2, type: "StepCompleted", body: { type: "StepCompleted", seq: 2, at, stepId: "evaluateRules", output: { ref: `inline:${JSON.stringify(evaluated)}` } } }, - ]; - let read = log.slice(0, 1); - const reader = { - async readRunEvents() { return read; }, - async readLatestRunEvents(_repo: unknown, _ref: string, include: (id: string) => boolean) { - return { tip: "t", events: new Map(include("run_backlog") ? [["run_backlog", read.at(-1)!]] : []) }; - }, - } as unknown as WorkflowRunReader; - const observeRuns = createTriageRuns({ runReader: reader, readSettled: async () => new Map(), maxKnownRuns: 100 }); - expect((await observeRuns(ANCHOR, DOMAIN)).verdicts).toEqual([]); - read = log; - const { verdicts } = await observeRuns(ANCHOR, DOMAIN); - expect(verdicts.map(({ number, headSha, state, detail }) => ({ number, headSha, state, reason: detail?.reason }))).toEqual([ - { number: 2, headSha: "sha2", state: "ready-monitoring", reason: "every rule passed, and the policy has no decision-model rule" }, - ]); - }); -}); - -describe("verdicts the decision model was never asked about", () => { - const pack: CheckPack = { ...emptyPack(REPO), checks: { "no-merge-conflicts": { enabled: true }, focused: { enabled: true } } }; - const at = (minute: number) => new Date(Date.UTC(2026, 9, 7, 1, minute)).toISOString(); - - // The hub stamps each run with its deployment's version as it reads the log. - const VERSION = "0.1.0-sha-1"; - - function plan(rows: PrTriageRow[], runs: ObservedRun[], minute: number) { - const byHead = new Map([["1@sha1", runs.map((run) => ({ ...run, workflowVersion: VERSION }))]]); - const repos = [{ name: REPO, prs: [{ number: 1, headSha: "sha1", updatedAt: at(-60), draft: false }], rows }]; - return planTenant({ repos, runs: new Map([[REPO, byHead]]), now: new Date(at(minute)), policy: DEFAULT_RECONCILE_POLICY, workflowVersion: VERSION }).get(REPO)!; - } - - test("are triaged again once and settle on a verdict that asked", async () => { - const skipped = (await observe([1], await batchVerdicts([1], new Set(), { pack, skipJudge: true }), "run_old", at(0))).get("1@sha1")!; - expect(skipped[0]!.unsettled).toBe("model not asked"); - const triaged: PrTriageRow = { number: 1, headSha: "sha1", status: "triaged", attempts: 0, runId: "run_old", workflowVersion: VERSION, firstSeenAt: at(-1), queuedAt: at(-1), updatedAt: at(1) }; - const redo = plan([triaged], skipped, 10); - expect(redo.enqueue.map((queued) => queued.reason)).toEqual(["model not asked"]); - const asked = (await observe([1], await batchVerdicts([1], new Set(), { pack }), "run_new", at(11))).get("1@sha1")!; - expect(asked[0]!.unsettled).toBeUndefined(); - const settled = plan(redo.rows, [...skipped, ...asked], 20); - expect(settled.rows[0]).toMatchObject({ status: "triaged", runId: "run_new" }); - expect(plan(settled.rows, [...skipped, ...asked], 30).enqueue).toEqual([]); - }); - - test("are triaged again once even after the head spent its attempts", async () => { - const skipped = (await observe([1], await batchVerdicts([1], new Set(), { pack, skipJudge: true }), "run_old", at(0))).get("1@sha1")!; - const capped: PrTriageRow = { number: 1, headSha: "sha1", status: "triaged", attempts: DEFAULT_RECONCILE_POLICY.maxAttempts, runId: "run_old", workflowVersion: VERSION, firstSeenAt: at(-1), queuedAt: at(-1), updatedAt: at(1) }; - const redo = plan([capped], skipped, 10); - expect(redo.enqueue.map((queued) => queued.reason)).toEqual(["model not asked"]); - expect(redo.rows[0]).toMatchObject({ status: "queued", attempts: 1 }); - }); - - test("unknown mergeability keeps its own reason though the judge was asked", async () => { - const reply = await batchVerdicts([1], new Set(), { pack, pr: { mergeable: null } }); - const verdict = JSON.parse(reply).items[0] as { verdict: string; checks: Array<{ kind: string; result: string }> }; - expect(verdict.verdict).toBe("unsure"); - expect(verdict.checks.filter((check) => check.kind === "model").map((check) => check.result)).toEqual(["pass"]); - const [run] = (await observe([1], reply)).get("1@sha1")!; - expect(run).toMatchObject({ unsettled: "unconfirmed checks: no-merge-conflicts", unconfirmed: true }); - }); - - test("deterministic and ambiguous human gates settle while a real inference outage requeues", async () => { - const pack: CheckPack = { ...emptyPack(REPO), checks: { focused: { enabled: true } } }; - function facts(files: Array<{ path: string; patch?: string }>) { - return { ...NO_FACTS, repo: REPO, number: 1, title: "Change", author: "octocat", headSha: "sha1", mergeable: true, checks: "success", files, paths: files.map((file) => file.path) }; - } - async function triage(files: Array<{ path: string; patch?: string }>, event?: ReactorInboundEvent) { - const ruled = await rules({ items: [{ facts: facts(files), pack }], batch: true }, ctx, signal); - const { output, reply } = await judgeAndEvaluate(ruled, () => event!); - return { ruled, judged: output?.map((chunk) => JSON.parse(chunk.reply) as Record), reply }; - } - - const compact = [{ path: "src/config.ts", patch: '@@ -0,0 +1 @@\n+const value = { name: "Candidate" };' }]; - const wide = Array.from({ length: 20 }, (_, index) => `@@ -${index + 1} +${index + 1} @@\n+// ${"設".repeat(3_000)}`).join("\n"); - const oversized = Array.from({ length: 40 }, (_, index) => ({ path: `src/wide${index}.ts`, patch: wide })); - const local = await triage(oversized); - const provider = await triage(compact, { type: "inference.error", error: { message: QUALITY_EVALUATION_LIMIT_ERROR } } as ReactorInboundEvent); - const ambiguousDecision = JSON.stringify({ - id: "focused", type: "choice", choice: "cant_tell", confidence: 0.9, - probabilities: { one_thing: 0.05, more_than_one_thing: 0.05, cant_tell: 0.9 }, - }) + JSON.stringify({ id: "instructed", type: "noul", noul: 0.1 }); - const ambiguousResult = await triage(compact, { type: "inference.done", turn: { content: [{ type: "text", text: ambiguousDecision }] } } as ReactorInboundEvent); - expect(local.judged).toBeUndefined(); - expect(local.ruled.items[0]).toMatchObject({ judgeError: expect.stringMatching(/^pull request too large for the decision model \(\d+ requests\)$/), judgeLimitExceeded: true }); - expect(provider.judged).toEqual([{ judgeError: QUALITY_EVALUATION_LIMIT_ERROR }]); - - const deterministic = (await observe([1], local.reply, "run_limit", at(0))).get("1@sha1")!; - const ambiguous = (await observe([1], ambiguousResult.reply, "run_ambiguous", at(0))).get("1@sha1")!; - const outage = (await observe([1], provider.reply, "run_outage", at(0))).get("1@sha1")!; - - expect(deterministic[0]?.unsettled).toBeUndefined(); - expect(ambiguous[0]?.unsettled).toBeUndefined(); - expect(plan([], deterministic, 10).enqueue).toEqual([]); - expect(plan([], ambiguous, 10).enqueue).toEqual([]); - expect(outage[0]?.unsettled).toBe(`degraded: decision model unavailable: ${QUALITY_EVALUATION_LIMIT_ERROR}`); - expect(plan([], outage, 10).enqueue.map((queued) => queued.reason)).toEqual([`degraded: decision model unavailable: ${QUALITY_EVALUATION_LIMIT_ERROR}`]); - }); -}); - -describe("a head CI is still running on", () => { - const pack: CheckPack = { ...emptyPack(REPO), checks: { "ci-green": { enabled: true } } }; - const at = (minute: number) => new Date(Date.UTC(2026, 9, 7, 1, minute)).toISOString(); - - function plan(rows: PrTriageRow[], runs: ObservedRun[], ci: CiState, minute: number) { - const repos = [{ name: REPO, prs: [{ number: 1, headSha: "sha1", updatedAt: at(-60), draft: false }], rows, ci: new Map([["1@sha1", ci]]) }]; - return planTenant({ repos, runs: new Map([[REPO, new Map([["1@sha1", runs]])]]), now: new Date(at(minute)), policy: DEFAULT_RECONCILE_POLICY, workflowVersion: undefined }).get(REPO)!; - } - - test("records no verdict and waits for CI, then is triaged once CI finishes and records the pass", async () => { - const held = await observeRuns([1], await batchVerdicts([1], new Set(), { pack, checks: [{ name: "build", status: "in_progress", conclusion: null }] }), "run_held", at(0)); - expect(held.verdicts).toEqual([]); - const runs = held.byRepo.get(REPO)!.get("1@sha1")!; - expect(runs).toMatchObject([{ status: "completed", heldForCi: true }]); - const running: PrTriageRow = { number: 1, headSha: "sha1", status: "running", attempts: 1, runId: "run_held", firstSeenAt: at(-1), queuedAt: at(-1), updatedAt: at(0), events: ["ready"] }; - const waiting = plan([running], runs, "pending", 2); - expect(waiting.rows[0]).toMatchObject({ status: "waiting", events: ["ready"] }); - expect(waiting.enqueue).toEqual([]); - expect(plan(waiting.rows, runs, "success", 4).enqueue).toEqual([{ number: 1, reason: "waited for CI", events: ["ready"] }]); - const passed = await observeRuns([1], await batchVerdicts([1], new Set(), { pack, checks: [{ name: "build", status: "completed", conclusion: "success" }] }), "run_pass", at(5)); - expect(passed.verdicts).toMatchObject([{ number: 1, headSha: "sha1", detail: { verdict: "pass" } }]); - }); - - test("a closed pull request records no verdict", async () => { - const closed = await observeRuns([1], await batchVerdicts([1], new Set(), { pr: { state: "closed" } }), "run_closed", at(0)); - expect(closed.verdicts).toEqual([]); - }); -}); diff --git a/apps/hub/src/github/coalescer.test.ts b/apps/hub/src/github/coalescer.test.ts index c24855ede..9607f95f9 100644 --- a/apps/hub/src/github/coalescer.test.ts +++ b/apps/hub/src/github/coalescer.test.ts @@ -375,7 +375,7 @@ test("a paused-label head whose pull request cannot be read is left failed, and expect(mails).toEqual([]); expect(state.rows()).toEqual([expect.objectContaining({ status: "failed", error: "delivery failed: Error: 502 Bad Gateway" })]); const pr = { number: 8, headSha: "sha1", updatedAt: new Date(START).toISOString(), draft: false }; - const plan = planTenant({ repos: [{ name: REPO, prs: [pr], rows: state.rows() }], runs: new Map(), now: new Date(START + DEFAULT_RECONCILE_POLICY.backoffMaxMs), policy: DEFAULT_RECONCILE_POLICY, workflowVersion: undefined }); + const plan = planTenant({ repos: [{ name: REPO, prs: [pr], rows: state.rows() }], runs: new Map(), now: new Date(START + DEFAULT_RECONCILE_POLICY.backoffMaxMs), policy: DEFAULT_RECONCILE_POLICY }); expect(plan.get(REPO)?.enqueue.map((entry) => entry.number)).toEqual([8]); }); diff --git a/apps/hub/src/github/deployment.ts b/apps/hub/src/github/deployment.ts index bc07dee92..09a3698d0 100644 --- a/apps/hub/src/github/deployment.ts +++ b/apps/hub/src/github/deployment.ts @@ -1,4 +1,4 @@ -import { and, desc, eq, gt, inArray, isNotNull, isNull, ne, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; import { schema, type DB, type ResolvedSenderPrincipal, type WorkflowRunLaunchSpecStore } from "@intx/db"; import { MAIL_ADMISSION_DENIED_CODE } from "@intx/hub-sessions"; import type { RunTriggerMaterialize } from "@corbits/webhooks"; @@ -50,12 +50,8 @@ export function systemSenderMaterializer(materialize: RunTriggerMaterialize, sen }; } -/** - * `cancelling` once cancellation was requested: the run stays live until the lifecycle sweep stops it. `workflowVersion` - * once read from its launch spec, for one pinned from the package registry; `offeringId`, the decision-model offering - * the deployment resolved, once read from its launch spec. - */ -export type LiveDeployment = { runId: string; address: string; createdAt: Date; cancelling: boolean; endedAt?: Date; workflowVersion?: string; offeringId?: string }; +/** `cancelling` once cancellation was requested: the run stays live until the lifecycle sweep stops it. */ +export type LiveDeployment = { runId: string; address: string; createdAt: Date; cancelling: boolean }; export type TenantDeployment = LiveDeployment & { workflow: string }; @@ -72,7 +68,7 @@ function markedAs(kind: DeploymentKind) { return kind === "worker" ? eq(marker, "true") : isNull(marker); } -type DeploymentQuery = { tenantId: string; definitionName?: string; kind: DeploymentKind; endedSince?: Date }; +type DeploymentQuery = { tenantId: string; definitionName?: string; kind: DeploymentKind }; /** * A workflow the portal publishes per person deploys from an asset named for its publisher and release, which names @@ -80,10 +76,9 @@ type DeploymentQuery = { tenantId: string; definitionName?: string; kind: Deploy */ export const workflowName = sql`coalesce(${schema.workflowRunLaunchSpec.frozenApprovalBundle} -> 'projection' ->> 'id', ${schema.workflowDefinition.name})`; -/** Matches how the hub lists deployments: a deployment is its anchor run. Newest first; live, or ended since `endedSince`. */ +/** Matches how the hub lists deployments: a deployment is its anchor run. Newest first, live only. */ async function deploymentsWhere(db: DB["db"], query: DeploymentQuery): Promise { const { workflowRun, workflowDefinition, workflowRunLaunchSpec, tenant, liveWorkflowRunStatuses } = schema; - const live = inArray(workflowRun.status, [...liveWorkflowRunStatuses]); const rows = await db .select({ runId: workflowRun.id, @@ -91,7 +86,6 @@ async function deploymentsWhere(db: DB["db"], query: DeploymentQuery): Promise { - return deploymentsWhere(db, { tenantId, definitionName, kind: "worker", endedSince }); -} - -export type ActiveTenant = { id: string; domain: string }; - -/** - * Tenants with a child run of the definition in flight, or ended or created since `endedSince`: a run inserted already - * completed, as a Bitbucket run is, keeps a null `endedAt` (stock leaves a terminal run routable that way), so its - * creation time is what says it just ended, and its verdict is recorded within a pass. - */ -export async function activeTenants(db: DB["db"], definitionName: string, what: "runs" | "deployments", endedSince?: Date): Promise { - const { workflowRun, workflowDefinition, tenant, liveWorkflowRunStatuses } = schema; - const running = eq(workflowRun.status, "running"); - const childRuns = endedSince === undefined ? running : or(running, gt(workflowRun.endedAt, endedSince), gt(workflowRun.createdAt, endedSince)); - return db - .selectDistinct({ id: tenant.id, domain: tenant.domain }) - .from(workflowRun) - .innerJoin(workflowDefinition, eq(workflowRun.definitionId, workflowDefinition.id)) - .innerJoin(tenant, eq(workflowRun.tenantId, tenant.id)) - .where(and( - eq(workflowDefinition.name, definitionName), - what === "runs" ? ne(workflowRun.id, workflowRun.anchorRunId) : eq(workflowRun.id, workflowRun.anchorRunId), - what === "runs" ? childRuns : inArray(workflowRun.status, [...liveWorkflowRunStatuses]), - )); -} - export function newestLive(deployments: readonly LiveDeployment[]): LiveDeployment | undefined { return deployments.find((live) => !live.cancelling); } @@ -201,20 +166,6 @@ function pinFor(bundle: FrozenApprovalBundle): string | undefined { return pinOf(bundle.closure); } -export async function withWorkflowVersions( - launchSpecs: Pick, - deployments: readonly T[], -): Promise { - const versioned: T[] = []; - for (const deployment of deployments) { - const spec = await launchSpecs.get(deployment.runId); - const workflowVersion = spec ? closureVersions(spec.frozenApprovalBundle)?.workflow.version : undefined; - const offeringId = spec?.defaultSourceOfferingId ?? undefined; - versioned.push({ ...deployment, ...(workflowVersion !== undefined && { workflowVersion }), ...(offeringId !== undefined && { offeringId }) }); - } - return versioned; -} - /** The reason the hub gave when it replaced a busy deployment with a fresh copy, which earlier audit entries carry. */ export const ROTATION_REASON = "Replaced by a fresh deployment of the same workflow"; diff --git a/apps/hub/src/github/github-read-access.postgres.test.ts b/apps/hub/src/github/github-read-access.postgres.test.ts index 8bca1b62e..d16ce3e6a 100644 --- a/apps/hub/src/github/github-read-access.postgres.test.ts +++ b/apps/hub/src/github/github-read-access.postgres.test.ts @@ -308,7 +308,7 @@ test("PostgreSQL: a code review is sent once per head as one COMMENT review, inl test("PostgreSQL: every verdict route needs workflow-run read, which holding the GitHub credential does not give", async () => { const unused = {} as never; const app = guardedApp(handle.db, sessionOf, grants, { - triageVerdicts: createTriageVerdicts({ ...deps, list, verdicts: unused, observer: unused, ready: unused, versions: unused, openPulls: unused, triageState: unused, now: unused }), + triageVerdicts: createTriageVerdicts({ ...deps, list, verdicts: unused, managerRuns: unused, openPulls: unused, triageState: unused, now: unused }), triageTimeline: createTriageTimeline({ verdicts: unused }), triageStats: createTriageStats({ verdicts: unused, outcomes: unused, now: unused, log: unused }), audit: createAuditLog({ db: unused, artifacts: unused, readCheckPack: unused, runReader: unused, sameSource: unused, authorize: unused, workflows: ["pr-triage"] }), diff --git a/apps/hub/src/github/interleave.postgres.test.ts b/apps/hub/src/github/interleave.postgres.test.ts index 4ec7f102a..0012ee097 100644 --- a/apps/hub/src/github/interleave.postgres.test.ts +++ b/apps/hub/src/github/interleave.postgres.test.ts @@ -12,7 +12,7 @@ import { createGithubPrTriage } from "./pr-triage.js"; import { DEFAULT_RECONCILE_POLICY } from "./reconcile-plan.js"; import { createTriageReconciler } from "./triage-reconciler.js"; import { createTriageStateStore, type TriageStateStore } from "./triage-state-store.js"; -import type { ObservedRuns } from "./triage-runs.js"; +import type { ObservedRuns } from "./manager-runs.js"; /** No deployment holds the credential in these tests. */ async function noKeyPush(): Promise {} @@ -39,7 +39,7 @@ function stubDb() { } async function noRuns(): Promise { - return { byRepo: new Map(), runCount: 0, verdicts: [], approved: [] }; + return { byRepo: new Map() }; } function route(store: TriageStateStore, deliver: Deliver) { @@ -67,7 +67,6 @@ function reconciler(store: TriageStateStore, heads: number[], deliver: Deliver, listener: async () => DEPLOYMENT, openHeadsFor: async () => async () => heads.map((number) => ({ number, headSha: number === 8 ? "abc123" : `sha${number}`, updatedAt: LONG_AGO, draft: false })), activity: { stale: () => true }, - observeTenant: async () => ({ deployments: [DEPLOYMENT], observed: [await noRuns()], workers: [] }), managerRuns: noRuns, headChecks: async () => checks, verdicts: { latest: async () => [] }, diff --git a/apps/hub/src/github/manager-runs.postgres.test.ts b/apps/hub/src/github/manager-runs.postgres.test.ts index f6a52a4ea..ce4783f95 100644 --- a/apps/hub/src/github/manager-runs.postgres.test.ts +++ b/apps/hub/src/github/manager-runs.postgres.test.ts @@ -80,10 +80,10 @@ test("PostgreSQL: the reconciler does not mail a head again while a live manager const queued: PrTriageRow = { number: 7, headSha: "h1", status: "queued", attempts: 1, firstSeenAt: queuedAt, queuedAt, updatedAt: queuedAt }; function plan(rows: PrTriageRow[], runs: Awaited>) { const ci = new Map([[runKey(8, "h1"), "pending" as const]]); - return planTenant({ repos: [{ name: REPO, prs: [open(7, now), open(8, now)], rows, ci }], runs: runs.byRepo, now, policy: DEFAULT_RECONCILE_POLICY, workflowVersion: "1.0.0" }).get(REPO)!; + return planTenant({ repos: [{ name: REPO, prs: [open(7, now), open(8, now)], rows, ci }], runs: runs.byRepo, now, policy: DEFAULT_RECONCILE_POLICY }).get(REPO)!; } - const running = plan([queued], await managerRuns(TENANT, [pr7, pr8], "1.0.0")); + const running = plan([queued], await managerRuns(TENANT, [pr7, pr8])); expect(running.rows.find((row) => row.number === 7)).toMatchObject({ status: "running", runId: "run_triaged" }); expect(running.enqueue.map((queuedPr) => queuedPr.number)).not.toContain(7); @@ -92,9 +92,9 @@ test("PostgreSQL: the reconciler does not mail a head again while a live manager await take(db, TENANT, held, "run_held"); await done(db, TENANT, held, "run_held", [], { head: "h1", waiting: true }); const firstSeenAt = new Date(now.getTime() - MINUTE).toISOString(); - const fresh: PrTriageRow = { number: 8, headSha: "h1", status: "new", attempts: 0, workflowVersion: "1.0.0", firstSeenAt, updatedAt: firstSeenAt }; + const fresh: PrTriageRow = { number: 8, headSha: "h1", status: "new", attempts: 0, firstSeenAt, updatedAt: firstSeenAt }; - const settled = plan([...running.rows.filter((row) => row.number === 7), fresh], await managerRuns(TENANT, [pr7, pr8], "1.0.0")); + const settled = plan([...running.rows.filter((row) => row.number === 7), fresh], await managerRuns(TENANT, [pr7, pr8])); expect(settled.rows).toEqual([expect.objectContaining({ number: 7, status: "triaged", runId: "run_triaged" }), expect.objectContaining({ number: 8, status: "waiting", runId: "run_held" })]); expect(settled.enqueue).toEqual([]); }); @@ -111,8 +111,8 @@ test("PostgreSQL: on the listener path a head is mailed once while its manager r let mails = 0; async function reconcile(afterMs: number): Promise { now = new Date(start.getTime() + afterMs); - const runs = await managerRuns(TENANT, [pr], undefined); - const plan = planTenant({ repos: [{ name: REPO, prs: [open(7, start)], rows }], runs: runs.byRepo, now, policy: DEFAULT_RECONCILE_POLICY, workflowVersion: undefined }).get(REPO)!; + const runs = await managerRuns(TENANT, [pr]); + const plan = planTenant({ repos: [{ name: REPO, prs: [open(7, start)], rows }], runs: runs.byRepo, now, policy: DEFAULT_RECONCILE_POLICY }).get(REPO)!; rows = plan.rows; mails += plan.enqueue.length; return rows[0]!; @@ -143,11 +143,11 @@ test("PostgreSQL: a manager that went away mid-pass fails its pass's head, so th await take(db, TENANT, pr7, "run_gone"); await db.update(schema.workflowRun).set({ status: "failed" }).where(sql`${schema.workflowRun.id} = 'run_gone'`); - const runs = await managerRuns(TENANT, [pr7], undefined); + const runs = await managerRuns(TENANT, [pr7]); expect(runs.byRepo.get(REPO)?.get(runKey(7, "h1"))).toEqual([expect.objectContaining({ runId: "run_gone", status: "failed" })]); const queuedAt = new Date(now.getTime() - MINUTE).toISOString(); const queued: PrTriageRow = { number: 7, headSha: "h1", status: "queued", attempts: 1, firstSeenAt: queuedAt, queuedAt, updatedAt: queuedAt }; - const plan = planTenant({ repos: [{ name: REPO, prs: [open(7, now)], rows: [queued] }], runs: runs.byRepo, now, policy: DEFAULT_RECONCILE_POLICY, workflowVersion: undefined }).get(REPO)!; + const plan = planTenant({ repos: [{ name: REPO, prs: [open(7, now)], rows: [queued] }], runs: runs.byRepo, now, policy: DEFAULT_RECONCILE_POLICY }).get(REPO)!; expect(plan.rows).toEqual([expect.objectContaining({ status: "failed", runId: "run_gone", error: "run failed" })]); expect(plan.enqueue).toEqual([]); }); diff --git a/apps/hub/src/github/manager-runs.ts b/apps/hub/src/github/manager-runs.ts index a35014f88..49531d1e3 100644 --- a/apps/hub/src/github/manager-runs.ts +++ b/apps/hub/src/github/manager-runs.ts @@ -6,11 +6,13 @@ import type { DB } from "@intx/db"; import { unsettledBy } from "@corbits/triage-workflows/logic/settled"; import { managedPulls, type PullRequestRef } from "../manager-state.js"; import { runKey, type ObservedRun } from "./reconcile-plan.js"; -import type { ObservedRuns } from "./triage-runs.js"; import type { VerdictRecord, VerdictRecordStore } from "./verdict-record-store.js"; -/** The runs of `pulls`' managers; `workflowVersion` is the version new mail goes to, which a manager's runs count as. */ -export type ManagerRuns = (tenantId: string, pulls: readonly PullRequestRef[], workflowVersion: string | undefined) => Promise; +/** Runs by repository, then by `${number}@${headSha}`. */ +export type ObservedRuns = { byRepo: Map> }; + +/** The runs of `pulls`' managers. */ +export type ManagerRuns = (tenantId: string, pulls: readonly PullRequestRef[]) => Promise; function settlesBy(verdict: VerdictRecord): Pick { const { detail } = verdict; @@ -18,10 +20,9 @@ function settlesBy(verdict: VerdictRecord): Pick, now: () => Date): ManagerRuns { - return async function managerRuns(tenantId, asked, workflowVersion) { + return async function managerRuns(tenantId, asked) { const pulls = await managedPulls(db, tenantId, asked); const recorded = await verdicts.ofRuns(tenantId, [...new Set(pulls.flatMap((pull) => pull.history.map((pass) => pass.runId)))], asked); - const stamp = workflowVersion === undefined ? {} : { workflowVersion }; const byRepo: ObservedRuns["byRepo"] = new Map(); function observed(repo: string, number: number, head: string, run: ObservedRun): void { @@ -29,14 +30,14 @@ export function createManagerRuns(db: DB["db"], verdicts: Pick [pass.head, pass])); @@ -47,6 +48,6 @@ export function createManagerRuns(db: DB["db"], verdicts: Pick { test("a second click before the run starts is refused once the hub has passed over the queued head", async () => { const at = (agoMs: number) => new Date(NOW.getTime() - agoMs).toISOString(); - const oldVerdict: ObservedRun = { runId: "run-old", source: "github", status: "completed", startedAt: at(3_600_000), workflowVersion: "0.1.0-sha-1" }; - let rows: PrTriageRow[] = [{ number: 8, headSha: "abc123", status: "triaged", attempts: 1, runId: "run-old", workflowVersion: "0.1.0-sha-1", firstSeenAt: at(7_200_000), queuedAt: at(3_600_000), updatedAt: at(3_600_000) }]; + const oldVerdict: ObservedRun = { runId: "run-old", source: "github", status: "completed", startedAt: at(3_600_000) }; + let rows: PrTriageRow[] = [{ number: 8, headSha: "abc123", status: "triaged", attempts: 1, runId: "run-old", firstSeenAt: at(7_200_000), queuedAt: at(3_600_000), updatedAt: at(3_600_000) }]; const s = stub(); const store: GithubPrTriageDeps["store"] = { async load() { return { rows, version: VERSION }; }, async save(_t, _r, next) { rows = next; return VERSION; } }; const deps = { store, observeRuns: observed([oldVerdict]) }; @@ -262,7 +262,7 @@ describe("createGithubPrTriage", () => { rows = planTenant({ repos: [{ name: REPO, prs: [{ number: 8, headSha: "abc123", updatedAt: at(3_600_000), draft: false }], rows }], runs: new Map([[REPO, new Map([["8@abc123", [oldVerdict]]])]]), - now: NOW, policy: DEFAULT_RECONCILE_POLICY, workflowVersion: "0.1.0-sha-1", + now: NOW, policy: DEFAULT_RECONCILE_POLICY, }).get(REPO)!.rows; const second = await handler(ENABLED, s, deps)(request(), MEMBER); expect({ status: second.status, delivered: s.delivered.length, row: rows[0]!.status }).toEqual({ status: 409, delivered: 1, row: "queued" }); diff --git a/apps/hub/src/github/pr-triage.ts b/apps/hub/src/github/pr-triage.ts index 762ebff5a..4df116292 100644 --- a/apps/hub/src/github/pr-triage.ts +++ b/apps/hub/src/github/pr-triage.ts @@ -17,7 +17,7 @@ import type { ManagerRuns } from "./manager-runs.js"; import { isStuck, runKey, type ObservedRun, type ReconcilePolicy } from "./reconcile-plan.js"; import { repoRecords, sourceOf, triageNs, triageStateKey, type RepoRecord } from "./tenant-config.js"; import type { LiveDeployment } from "./deployment.js"; -import type { ObservedRuns } from "./triage-runs.js"; +import type { ObservedRuns } from "./manager-runs.js"; import { isQueuedSince, isRunningSince, queueHead, restoreHead } from "./triage-queue.js"; import type { TriageStateStore } from "./triage-state-store.js"; @@ -105,7 +105,7 @@ export function createGithubPrTriage(deps: GithubPrTriageDeps) { const now = deps.now(); const key = runKey(number, headSha); - const managed = headRuns(await deps.managerRuns(tenantId, [{ source: sourceOf(record), repo, number }], undefined), repo, key); + const managed = headRuns(await deps.managerRuns(tenantId, [{ source: sourceOf(record), repo, number }]), repo, key); if (managed.some((run) => run.status === "running" && !isStuck(run, now, deps.policy))) { return failure(409, "already_running", `${repo}#${number} is already being triaged.`); } diff --git a/apps/hub/src/github/reconcile-plan.test.ts b/apps/hub/src/github/reconcile-plan.test.ts index b0a35de86..b7c127ef3 100644 --- a/apps/hub/src/github/reconcile-plan.test.ts +++ b/apps/hub/src/github/reconcile-plan.test.ts @@ -15,30 +15,20 @@ function pr(number: number, headSha = `sha${number}`): OpenPr { return { number, headSha, updatedAt: at(-1_000).toISOString(), draft: false }; } -function run(runId: string, status: ObservedRun["status"], startedMinute: number, workflowVersion = VERSION): ObservedRun { - return { runId, source: "github", status, startedAt: at(startedMinute).toISOString(), workflowVersion }; +function run(runId: string, status: ObservedRun["status"], startedMinute: number): ObservedRun { + return { runId, source: "github", status, startedAt: at(startedMinute).toISOString() }; } const REPO = "acme/widgets"; -const VERSION = "0.1.0-sha-22222222"; -const OLDER = "0.1.0-sha-11111111"; -const NEWER = "0.1.0-sha-33333333"; - -function plan(prs: OpenPr[], rows: readonly PrTriageRow[], now: Date, runs: Array<[OpenPr, ObservedRun[]]> = [], maxInFlight = prs.length, workflowVersion: string | null = VERSION): RepoPlan { +function plan(prs: OpenPr[], rows: readonly PrTriageRow[], now: Date, runs: Array<[OpenPr, ObservedRun[]]> = [], maxInFlight = prs.length): RepoPlan { const byHead = new Map(runs.map(([open, observed]) => [runKey(open.number, open.headSha), observed])); - const input = { repos: [{ name: REPO, prs, rows }], runs: new Map([[REPO, byHead]]), now, policy: { ...policy, maxInFlight }, workflowVersion: workflowVersion ?? undefined }; + const input = { repos: [{ name: REPO, prs, rows }], runs: new Map([[REPO, byHead]]), now, policy: { ...policy, maxInFlight } }; return planTenant(input).get(REPO)!; } -/** `null` is one from before verdicts were stamped. */ -function row(number: number, extra: Partial, workflowVersion: string | number | null = VERSION): PrTriageRow { - const base = { number, headSha: `sha${number}`, status: "new" as const, attempts: 0, firstSeenAt: at(-500).toISOString(), updatedAt: at(-500).toISOString() }; - return { ...base, ...(workflowVersion !== null && { workflowVersion }), ...extra }; -} - -function triaged(number: number, workflowVersion: string | number | null = VERSION): PrTriageRow { - return row(number, { status: "triaged", runId: `old${number}` }, workflowVersion); +function row(number: number, extra: Partial): PrTriageRow { + return { number, headSha: `sha${number}`, status: "new", attempts: 0, firstSeenAt: at(-500).toISOString(), updatedAt: at(-500).toISOString(), ...extra }; } function enqueued(result: RepoPlan): number[] { @@ -140,33 +130,6 @@ describe("reconcile plan", () => { expect(result.enqueue[0]?.reason).toBe("This repository still needs a policy."); }); - test("a workflow bump keeps settled verdicts, a legacy integer version included", () => { - const rows = [triaged(1, OLDER), triaged(2, 3)]; - const kept = plan([pr(1), pr(2)], rows, at(0)); - expect(enqueued(kept)).toEqual([]); - expect(kept.rows).toEqual(rows); - }); - - test("a deployment not pinned from the package registry resets nothing and stamps no version", () => { - const capped = row(1, { status: "failed", attempts: policy.maxAttempts, queuedAt: at(-500).toISOString() }, OLDER); - expect(enqueued(plan([pr(1)], [capped], at(100_000), [], 1, null))).toEqual([]); - expect(plan([pr(1)], [], at(0), [], 1, null).rows[0]?.workflowVersion).toBeUndefined(); - }); - - test("rows from before verdicts were stamped are not triaged again once the deployment is versioned", () => { - const legacy = plan([pr(1)], [triaged(1, null)], at(0)); - expect(enqueued(legacy)).toEqual([]); - expect(legacy.rows[0]).toEqual(triaged(1, null)); - }); - - test("a head in flight on the old version settles on its run and keeps it", () => { - const running = row(1, { status: "running", runId: "old1", queuedAt: at(-10).toISOString(), updatedAt: at(-5).toISOString() }, OLDER); - const old: Array<[OpenPr, ObservedRun[]]> = [[pr(1), [run("old1", "completed", -5, OLDER)]]]; - const settled = plan([pr(1)], [running], at(0), old); - expect(settled.rows[0]).toMatchObject({ status: "triaged", runId: "old1", workflowVersion: OLDER }); - expect(enqueued(plan([pr(1)], settled.rows, at(1), old))).toEqual([]); - }); - test("a verdict waiting on an unconfirmed check is retried after its own delay, up to the attempt cap", () => { const unconfirmed = (runId: string, minute: number): ObservedRun => ({ ...run(runId, "completed", minute), unsettled: "unconfirmed checks: conflicts", unconfirmed: true }); const runs: ObservedRun[] = [unconfirmed("run_w", 0)]; @@ -187,19 +150,6 @@ describe("reconcile plan", () => { expect(rows[0]).toMatchObject({ status: "failed", attempts: policy.maxAttempts }); }); - test("a head capped under old code is triaged again once a redeployed workflow runs a newer version", () => { - const capped: PrTriageRow = { number: 1, headSha: "sha1", status: "failed", attempts: policy.maxAttempts, workflowVersion: VERSION, firstSeenAt: at(0).toISOString(), queuedAt: at(5).toISOString(), updatedAt: at(5).toISOString() }; - expect(enqueued(plan([pr(1)], [capped], at(100_000)))).toEqual([]); - // The redeployed workflow is a new deployment: its log holds none of the old runs. - const upgraded = plan([pr(1)], [capped], at(100_000), [], 1, NEWER); - expect(enqueued(upgraded)).toEqual([1]); - expect(upgraded.rows[0]).toMatchObject({ status: "queued", attempts: 1, workflowVersion: NEWER }); - expect(upgraded.enqueue[0]?.reason).toBe(`attempts from workflow version ${VERSION}, current is ${NEWER}`); - const triaged = plan([pr(1)], upgraded.rows, at(100_010), [[pr(1), [run("run_new", "completed", 100_001, NEWER)]]], 1, NEWER); - expect(triaged.rows[0]).toMatchObject({ status: "triaged", runId: "run_new", workflowVersion: NEWER }); - expect(enqueued(plan([pr(1)], triaged.rows, at(100_020), [], 1, NEWER))).toEqual([]); - }); - test("a run started on a sidecar whose clock trails the hub still matches the hub's queue", () => { const queued = plan([pr(1)], [], at(0)).rows; const result = plan([pr(1)], queued, at(3), [[pr(1), [{ ...run("run_a", "failed", 0), startedAt: at(-0.5).toISOString() }]]]); @@ -239,18 +189,11 @@ describe("reconcile plan", () => { }); test("never-triaged heads are queued before heads due again, even when they were seen later", () => { - const stale = [1, 2, 3, 4].map((number) => row(number, { firstSeenAt: at(-100 + number).toISOString(), error: "attempts from workflow version 1, current is 2" })); + const stale = [1, 2, 3, 4].map((number) => row(number, { status: "failed", attempts: 1, firstSeenAt: at(-100 + number).toISOString(), queuedAt: at(-90).toISOString(), error: "degraded: facts interrupted: abort" })); const result = plan([pr(5), pr(6), pr(7), ...stale.map((row) => pr(row.number))], stale, at(0), [], policy.maxInFlight); expect(enqueued(result)).toEqual([5, 6, 7, 1, 2]); }); - test("a never-attempted head whose mail failed to deliver still goes before version-stale heads", () => { - const stale = [1, 2, 3].map((number) => row(number, { firstSeenAt: at(-100 + number).toISOString(), error: "attempts from workflow version 1, current is 2" })); - const undelivered = row(9, { firstSeenAt: at(-5).toISOString(), error: "delivery failed: Error: unroutable" }); - const result = plan([pr(9), pr(1), pr(2), pr(3)], [undelivered, ...stale], at(0), [], 1); - expect(enqueued(result)).toEqual([9]); - }); - test("a run still going for a superseded head, a closed pull request or an unloaded repository holds its slot", () => { const running = [run("r_live", "running", -5)]; const runs = new Map([ @@ -259,7 +202,7 @@ describe("reconcile plan", () => { ]); const rows = [row(1, { headSha: "sha1-old", status: "running", runId: "r_live", queuedAt: at(-6).toISOString() })]; const prs = [pr(1, "sha1-new"), pr(2), pr(3), pr(4), pr(5), pr(6)]; - const plans = planTenant({ repos: [{ name: REPO, prs, rows }], runs, now: at(0), policy, workflowVersion: VERSION }); + const plans = planTenant({ repos: [{ name: REPO, prs, rows }], runs, now: at(0), policy }); expect(enqueued(plans.get(REPO)!)).toEqual([1, 2]); }); @@ -315,8 +258,8 @@ describe("reconcile plan", () => { }); describe("triage again on a head that already has a verdict", () => { - // The route's write on top of a triaged row: queued, queuedAt now, runId and workflowVersion kept. - const requeued: PrTriageRow = { number: 8, headSha: "sha8", status: "queued", attempts: 0, runId: "run-old", workflowVersion: VERSION, firstSeenAt: at(-120).toISOString(), queuedAt: at(-1).toISOString(), updatedAt: at(-1).toISOString() }; + // The route's write on top of a triaged row: queued, queuedAt now, runId kept. + const requeued: PrTriageRow = { number: 8, headSha: "sha8", status: "queued", attempts: 0, runId: "run-old", firstSeenAt: at(-120).toISOString(), queuedAt: at(-1).toISOString(), updatedAt: at(-1).toISOString() }; const oldVerdict = run("run-old", "completed", -60); test("the next pass keeps the head queued until the maintainer's run starts", () => { @@ -332,7 +275,7 @@ describe("a head waiting for CI", () => { const waiting = row(1, { status: "waiting", firstSeenAt: at(0).toISOString(), queuedAt: at(0).toISOString(), events: ["opened"] }); function release(ci: CiState | undefined, minute: number): RepoPlan { - const input = { repos: [{ name: REPO, prs: [pr(1)], rows: [waiting], ci: new Map(ci === undefined ? [] : [[runKey(1, "sha1"), ci]]) }], runs: new Map(), now: at(minute), policy, workflowVersion: VERSION }; + const input = { repos: [{ name: REPO, prs: [pr(1)], rows: [waiting], ci: new Map(ci === undefined ? [] : [[runKey(1, "sha1"), ci]]) }], runs: new Map(), now: at(minute), policy }; return planTenant(input).get(REPO)!; } @@ -353,7 +296,7 @@ describe("a head waiting for CI", () => { test("a head no webhook queued waits for its running CI too", () => { function seen(ci: CiState): RepoPlan { - return planTenant({ repos: [{ name: REPO, prs: [pr(2)], rows: [], ci: new Map([[runKey(2, "sha2"), ci]]) }], runs: new Map(), now: at(0), policy, workflowVersion: VERSION }).get(REPO)!; + return planTenant({ repos: [{ name: REPO, prs: [pr(2)], rows: [], ci: new Map([[runKey(2, "sha2"), ci]]) }], runs: new Map(), now: at(0), policy }).get(REPO)!; } expect(enqueued(seen("pending"))).toEqual([]); expect(status(seen("pending"), 2)?.status).toBe("waiting"); diff --git a/apps/hub/src/github/reconcile-plan.ts b/apps/hub/src/github/reconcile-plan.ts index ef9ea8a19..8c8ab0290 100644 --- a/apps/hub/src/github/reconcile-plan.ts +++ b/apps/hub/src/github/reconcile-plan.ts @@ -1,9 +1,8 @@ -// Decides, from what GitHub and the run logs say now, which open pull requests -// to queue again. Pure, so every recovery path is decided in one place. +// Decides, from what the host and the managers' runs say now, which open pull +// requests to queue again. Pure, so every recovery path is decided in one place. import { PR_TRIAGE_STUCK_RUN_MS, type PrTriageRow, type RepoSource, type TriageEvent } from "@corbits/triage-contracts"; import type { CiState } from "@corbits/github-reads"; import { MODEL_NOT_ASKED } from "@corbits/triage-workflows/logic/settled"; -import type { Judged } from "@corbits/triage-workflows/logic/item"; /** `labels` only where the host lists them. */ export type OpenPr = { number: number; headSha: string; updatedAt: string; draft: boolean; labels?: string[] }; @@ -20,11 +19,9 @@ export type ObservedRun = { unconfirmed?: true; /** The run waited for CI still running on the head instead of deciding, so the head waits for its CI. */ heldForCi?: true; - /** Workflow package version of the deployment the run ran on; the row keeps it to reset failed heads on a later deployment and to carry judge answers. */ - workflowVersion?: string; trigger?: string; - offeringId?: string; - judged?: Judged; + /** When a running manager took the pass it has open; absent while it is up between passes, as while it waits on a reviewer. */ + passAt?: string; }; export type ReconcilePolicy = { @@ -101,18 +98,14 @@ type RepoPlanInput = { export type TenantPlanInput = { repos: readonly RepoPlanInput[]; - /** Every observed run of the deployment, by repository, then by `${number}@${headSha}`, whether or not its repository or head is still here. */ + /** Every observed run, by repository, then by `${number}@${headSha}`, whether or not its repository or head is still here. */ runs: ReadonlyMap>; now: Date; policy: ReconcilePolicy; - /** Workflow package version of the deployment new mail goes to; none when it was not deployed from the package registry. */ - workflowVersion: string | undefined; - /** Decision-model offering of the deployment new mail goes to; judge answers are carried only between runs on the same one. */ - offeringId?: string; }; -/** A head to queue, why, the judge answers its earlier runs got, the events it waited on CI with, and whether it is decided while the host's checks are pending. */ -export type QueuedPr = { number: number; reason: string; judged?: Judged; events?: TriageEvent[]; noWait?: true }; +/** A head to queue, why, the events it waited on CI with, and whether it is decided while the host's checks are pending. */ +export type QueuedPr = { number: number; reason: string; events?: TriageEvent[]; noWait?: true }; /** Rows for exactly the open heads; queued heads are already marked queued. */ export type RepoPlan = { rows: PrTriageRow[]; enqueue: QueuedPr[] }; @@ -137,16 +130,14 @@ function backoffMs(attempts: number, policy: ReconcilePolicy): number { type Observation = { status: PrTriageRow["status"]; run?: ObservedRun; runId?: string; error?: string }; function withStatus(row: PrTriageRow, { status, run, runId = run?.runId, error }: Observation, now: string): PrTriageRow { - const workflowVersion = run?.workflowVersion ?? row.workflowVersion; - if (row.status === status && row.runId === runId && row.error === error && row.workflowVersion === workflowVersion) return row; - const { runId: _run, error: _error, workflowVersion: _version, events, ...rest } = row; + if (row.status === status && row.runId === runId && row.error === error) return row; + const { runId: _run, error: _error, events, ...rest } = row; return { ...rest, status, ...(events !== undefined && (inFlight({ status }) || status === "waiting") && { events }), ...(runId !== undefined && { runId }), ...(error !== undefined && { error }), - ...(workflowVersion !== undefined && { workflowVersion }), updatedAt: now, }; } @@ -246,17 +237,8 @@ function isDue(row: PrTriageRow, pr: OpenPr, runs: readonly ObservedRun[], ci: C return now.getTime() >= Math.max(last + backoffMs(row.attempts, policy), unconfirmedUntil); } -const STALE_VERDICT = "attempts from workflow version"; - -/** A head capped under another workflow version starts over, once: its attempts were spent on other code. A settled verdict is kept; the next event triages the head on the current version. A row that names no version is not stale; a legacy integer always differs. */ -function fresh(pr: OpenPr, prior: PrTriageRow | undefined, workflowVersion: string | undefined, at: string): PrTriageRow { - if (prior === undefined) { - return { number: pr.number, headSha: pr.headSha, status: "new", attempts: 0, ...(workflowVersion !== undefined && { workflowVersion }), firstSeenAt: at, updatedAt: at }; - } - if (workflowVersion === undefined || prior.workflowVersion === undefined || prior.workflowVersion === workflowVersion) return prior; - if (prior.status !== "failed") return prior; - const { runId: _run, queuedAt: _queued, ...rest } = prior; - return { ...rest, status: "new", attempts: 0, workflowVersion, error: `${STALE_VERDICT} ${prior.workflowVersion}, current is ${workflowVersion}`, updatedAt: at }; +function fresh(pr: OpenPr, prior: PrTriageRow | undefined, at: string): PrTriageRow { + return prior ?? { number: pr.number, headSha: pr.headSha, status: "new", attempts: 0, firstSeenAt: at, updatedAt: at }; } /** A queued head counts until the hub marks it failed for never starting, so a backed-up queue is not mailed again. */ @@ -279,28 +261,11 @@ function runningHeads(runs: TenantPlanInput["runs"], now: Date, policy: Reconcil return busy; } -/** Once a head is on the live version, a run of any other, such as one on a deployment still being cancelled, neither settles nor fails it; it still holds its slot. A head still on an older version keeps every run, so one in flight settles. */ -function counts(run: ObservedRun, row: PrTriageRow, workflowVersion: string | undefined): boolean { - return workflowVersion === undefined || row.workflowVersion !== workflowVersion || run.workflowVersion === workflowVersion; -} - -type Due = { repo: string; index: number; row: PrTriageRow; judged: Judged }; - -/** - * Every answer the head's completed runs got on the same workflow version and decision-model offering as the - * deployment mailed, a newer run's winning; none when either is unknown. The judge is asked only what none answered. - */ -function judgedOf(runs: readonly ObservedRun[], workflowVersion: string | undefined, offeringId: string | undefined): Judged { - if (workflowVersion === undefined || offeringId === undefined) return {}; - const done = runs - .filter((run) => run.status === "completed" && run.workflowVersion === workflowVersion && run.offeringId === offeringId) - .sort((a, b) => ms(a.startedAt) - ms(b.startedAt)); - return Object.assign({}, ...done.map((run) => run.judged ?? {})); -} +type Due = { repo: string; index: number; row: PrTriageRow }; -/** A head with no run and no verdict, even if its mail failed to deliver; a version-stale reset also has no run, so only its reason tells it apart. */ +/** A head with no run and no verdict, even if its mail failed to deliver. */ function neverTriaged(row: PrTriageRow): boolean { - return row.status === "new" && row.runId === undefined && !row.error?.startsWith(STALE_VERDICT); + return row.status === "new" && row.runId === undefined; } /** Never-attempted heads first, then oldest waiting first, a retried head by the time it was last queued; among equals, the oldest pull request first. */ @@ -310,7 +275,7 @@ function byWait(a: Due, b: Due): number { || a.row.number - b.row.number; } -/** Queues a settled head again for `reason`, keeping its attempts since it did not fail; the row keeps its run so the verdict's answers are carried. */ +/** Queues a settled head again for `reason`, keeping its attempts since it did not fail. */ export function requeue(plan: RepoPlan, row: PrTriageRow, reason: string, now: Date, policy: ReconcilePolicy): void { const index = plan.rows.findIndex((candidate) => candidate.number === row.number && candidate.headSha === row.headSha); if (index === -1) return; @@ -321,7 +286,7 @@ export function requeue(plan: RepoPlan, row: PrTriageRow, reason: string, now: D } /** Queues due heads across the tenant's repositories, never-triaged ones first, while fewer than `maxInFlight` heads are queued or running. */ -export function planTenant({ repos, runs, now, policy, workflowVersion, offeringId }: TenantPlanInput): Map { +export function planTenant({ repos, runs, now, policy }: TenantPlanInput): Map { const at = now.toISOString(); const plans = new Map(); const due: Due[] = []; @@ -332,26 +297,26 @@ export function planTenant({ repos, runs, now, policy, workflowVersion, offering const next: PrTriageRow[] = []; for (const pr of prs) { const key = runKey(pr.number, pr.headSha); - const current = fresh(pr, byHead.get(key), workflowVersion, at); - const runsOfHead = (observed?.get(key) ?? []).filter((run) => counts(run, current, workflowVersion)); + const current = fresh(pr, byHead.get(key), at); + const runsOfHead = observed?.get(key) ?? []; const seen = uncapped(observe(current, runsOfHead, now, policy), now, policy); // A head no webhook queued waits for its running CI like one that did; one with no checks yet is taken as having none. const held = seen.status === "new" && ci?.get(key) === "pending" && isDue(seen, pr, runsOfHead, undefined, now, policy) && waitsForCi("pending", seen.firstSeenAt, now, policy); const row: PrTriageRow = held ? { ...seen, status: "waiting", updatedAt: at } : seen; if (inFlight(row)) busy.add(`${name} ${key}`); - if (isDue(row, pr, runsOfHead, ci?.get(key), now, policy)) due.push({ repo: name, index: next.length, row, judged: judgedOf(runsOfHead, workflowVersion, offeringId) }); + if (isDue(row, pr, runsOfHead, ci?.get(key), now, policy)) due.push({ repo: name, index: next.length, row }); next.push(row); } plans.set(name, { rows: next, enqueue: [] }); } - for (const { repo, index, row, judged } of due.sort(byWait).slice(0, Math.max(policy.maxInFlight - busy.size, 0))) { + for (const { repo, index, row } of due.sort(byWait).slice(0, Math.max(policy.maxInFlight - busy.size, 0))) { const plan = plans.get(repo)!; const { runId: _run, error: _error, events, ...rest } = row; // The events stay on the row while its run is in flight, so a check run finishing meanwhile is known to be covered by it. plan.rows[index] = { ...rest, status: "queued", ...(events !== undefined && { events }), attempts: row.attempts + 1, queuedAt: at, updatedAt: at }; const reason = row.status === "waiting" ? "waited for CI" : row.error ?? "never triaged"; const noWait = decideNow(row, row.attempts + 1, now, policy); - plan.enqueue.push({ number: row.number, reason, ...(Object.keys(judged).length > 0 && { judged }), ...(events !== undefined && { events }), ...(noWait && { noWait }) }); + plan.enqueue.push({ number: row.number, reason, ...(events !== undefined && { events }), ...(noWait && { noWait }) }); } return plans; } diff --git a/apps/hub/src/github/run-recorder.ts b/apps/hub/src/github/run-recorder.ts deleted file mode 100644 index 277bff3d0..000000000 --- a/apps/hub/src/github/run-recorder.ts +++ /dev/null @@ -1,50 +0,0 @@ -// Copies what the live deployments' run logs say into the verdict table, as the -// tenant observer reads them. -import type { ObservedRuns } from "./triage-runs.js"; -import { withApproved, type ApprovedDo, type RecordedVerdict, type VerdictRecordStore } from "./verdict-record-store.js"; - -export type RunRecorderDeps = { - verdicts: Pick; - log: (entry: Record) => void; -}; - -/** False when something could not be recorded; it is recorded again on the next read. */ -export type RecordRuns = (tenantId: string, runs: ObservedRuns) => Promise; - -/** A verdict is recorded once its step completes and again when its run posts the reply or runs its Dos. */ -function verdictState(verdict: RecordedVerdict): string { - return JSON.stringify(verdict); -} - -function approvalKey({ repo, number, effectId, status }: ApprovedDo): string { - return `${repo}#${number}\n${effectId}\n${status}`; -} - -/** The verdict with the outcomes of the Dos approved on its head, so recording it, or a re-triage of the head, keeps them. */ -function folded(verdict: RecordedVerdict, approved: readonly ApprovedDo[]): RecordedVerdict { - const own = approved.filter((done) => done.repo === verdict.repo && done.number === verdict.number); - return verdict.detail && own.length > 0 ? { ...verdict, detail: withApproved(verdict.detail, own) } : verdict; -} - -export function createRunRecorder(deps: RunRecorderDeps): RecordRuns { - /** Verdicts recorded per tenant that its live logs still hold, as recorded; a restart records them again, which changes nothing. */ - const recorded = new Map>(); - - const recordedApproved = new Map>(); - - return async function recordRuns(tenantId, runs) { - const seen = recorded.get(tenantId) ?? new Set(); - const seenApproved = recordedApproved.get(tenantId) ?? new Set(); - const verdicts = runs.verdicts.map((verdict) => folded(verdict, runs.approved)); - try { - await deps.verdicts.record(tenantId, verdicts.filter((verdict) => !seen.has(verdictState(verdict)))); - await deps.verdicts.recordApproved(tenantId, runs.approved.filter((done) => !seenApproved.has(approvalKey(done)))); - recorded.set(tenantId, new Set(verdicts.map(verdictState))); - recordedApproved.set(tenantId, new Set(runs.approved.map(approvalKey))); - return true; - } catch (err) { - deps.log({ level: "error", msg: "triage_verdicts_unrecorded", tenantId, error: String(err) }); - return false; - } - }; -} diff --git a/apps/hub/src/github/tenant-observer.test.ts b/apps/hub/src/github/tenant-observer.test.ts deleted file mode 100644 index 9a2e6693a..000000000 --- a/apps/hub/src/github/tenant-observer.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { expect, test } from "bun:test"; -import { createRecordPass, createTenantObserver, createTenantVersions } from "./tenant-observer.js"; -import type { ObservedRuns } from "./triage-runs.js"; - -const LIVE = { runId: "run_live", address: "run_live@t.test", createdAt: new Date(0), cancelling: false }; - -test("one read of a tenant's logs at a time, shared by every caller, and reused until the interval after it finished", async () => { - let now = 0; - let reads = 0; - let inFlight = 0; - let mostInFlight = 0; - let release: () => void = () => {}; - async function observeRuns(): Promise { - reads += 1; - inFlight += 1; - mostInFlight = Math.max(mostInFlight, inFlight); - await new Promise((resolve) => { - release = resolve; - }); - inFlight -= 1; - return { byRepo: new Map(), runCount: 0, verdicts: [], approved: [] }; - } - const observer = createTenantObserver({ - liveDeployments: async () => [LIVE], - observeRuns, - recordRuns: async () => true, - versions: createTenantVersions(), - minIntervalMs: 2_000, - now: () => new Date(now), - }); - - const reconciler = observer.observe("t", "t.test"); - const loop = observer.observe("t", "t.test"); - await Promise.resolve(); - await Promise.resolve(); - release(); - expect(await reconciler).toBe(await loop); - now = 1_999; - await observer.observe("t", "t.test"); - expect(reads).toBe(1); - - now = 2_000; - const fresh = observer.observe("t", "t.test"); - await Promise.resolve(); - await Promise.resolve(); - release(); - await fresh; - expect(reads).toBe(2); - expect(mostInFlight).toBe(1); -}); - -test("a tenant whose logs cannot be read is retried with backoff, logged once per step, and stays unread until a read succeeds", async () => { - let now = 0; - const reads: string[] = []; - const logged: unknown[] = []; - const bad = { id: "bad", domain: "bad.test" }; - const good = { id: "good", domain: "good.test" }; - const recording = createRecordPass({ - observer: { - async observe(tenantId) { - reads.push(tenantId); - if (tenantId === "bad") throw new Error("unreadable"); - return { deployments: [], observed: [], workers: [] }; - }, - }, - activeTenants: async (what) => (what === "deployments" ? [bad, good] : []), - endedWithinMs: 4_000, - backoffMs: 2_000, - maxBackoffMs: 8_000, - now: () => new Date(now), - log: (entry) => logged.push(entry["retryInMs"]), - }); - expect(recording.ready("good")).toBe(false); - for (; now <= 20_000; now += 2_000) await recording.pass(); - expect(recording.ready("good")).toBe(true); - expect(recording.ready("bad")).toBe(false); - // Read at 0, 2, 6 and 14 seconds: each failure doubles the wait, up to 8. - expect(reads.filter((tenant) => tenant === "bad")).toHaveLength(4); - expect(logged).toEqual([2_000, 4_000, 8_000, 8_000]); -}); - -test("the version moves when a verdict's detail changes, and again after a read whose recording failed", async () => { - let feedback = "Please rebase."; - let records = true; - const versions = createTenantVersions(); - const observer = createTenantObserver({ - liveDeployments: async () => [LIVE], - async observeRuns(): Promise { - const verdict = { - runId: "run_1", repo: "acme/widgets", number: 1, headSha: "sha1", startedAt: new Date(0), verdictAt: new Date(0), settled: true, - state: "needs-author-update", actor: "author", humanGated: false, degraded: null, checks: [], dos: [], commented: false, detail: { feedback }, - }; - return { byRepo: new Map(), runCount: 1, verdicts: [verdict], approved: [] }; - }, - recordRuns: async () => records, - versions, - minIntervalMs: 0, - now: () => new Date(0), - }); - await observer.observe("t", "t.test"); - await observer.observe("t", "t.test"); - expect(versions.get("t")).toBe(1); - feedback = "Please rebase onto main."; - records = false; - await observer.observe("t", "t.test"); - expect(versions.get("t")).toBe(2); - records = true; - await observer.observe("t", "t.test"); - expect(versions.get("t")).toBe(3); -}); diff --git a/apps/hub/src/github/tenant-observer.ts b/apps/hub/src/github/tenant-observer.ts deleted file mode 100644 index b47d37d3d..000000000 --- a/apps/hub/src/github/tenant-observer.ts +++ /dev/null @@ -1,179 +0,0 @@ -// One read of a tenant's live pr-triage logs at a time, shared by the -// reconciler and the recording loop, and recorded as it is read, so the -// verdict route only reads tables and the last observation. -import { createHash } from "node:crypto"; -import type { ActiveTenant, LiveDeployment } from "./deployment.js"; -import type { RecordRuns } from "./run-recorder.js"; -import { isRunning, mergeObservedRuns, observeDeployments, type ObserveRuns, type ObservedRuns } from "./triage-runs.js"; - -/** `observed` pairs with `deployments` by index; `workers` holds the runs of the coordinator's workers read with them. */ -export type TenantObservation = { deployments: LiveDeployment[]; observed: ObservedRuns[]; workers: ObservedRuns[] }; - -/** - * Bumped whenever what the portal is shown of a tenant may have changed; the verdict route's cache key. Kept in this process's - * memory, which holds because the hub runs as one replica: a second one would record without the other's route seeing it. - */ -export type TenantVersions = { bump(tenantId: string): void; get(tenantId: string): number }; - -export function createTenantVersions(): TenantVersions { - const versions = new Map(); - return { - bump(tenantId) { - versions.set(tenantId, (versions.get(tenantId) ?? 0) + 1); - }, - get(tenantId) { - return versions.get(tenantId) ?? 0; - }, - }; -} - -export type TenantObserverDeps = { - liveDeployments: (tenantId: string) => Promise; - /** The coordinator's workers, read for their one run and never mailed or rotated. */ - workers?: { - /** Worker deployments live or released since `endedSince`. */ - deployments: (tenantId: string, endedSince: Date) => Promise; - /** How long a released worker's log is still read: long enough for the next reconciler pass to see the run and settle its head. */ - endedWithinMs: number; - }; - observeRuns: ObserveRuns; - recordRuns: RecordRuns; - versions: TenantVersions; - minIntervalMs: number; - now: () => Date; -}; - -export type TenantObserver = { - observe(tenantId: string, domain: string): Promise; - last(tenantId: string): TenantObservation | undefined; -}; - -function fingerprint(runs: ObservedRuns): string { - const statuses = [...runs.byRepo.values()].flatMap((byHead) => [...byHead.values()].flat().map((run) => `${run.runId}:${run.status}`)); - return createHash("sha1").update(JSON.stringify([statuses.sort(), runs.verdicts])).digest("base64url"); -} - -export function createTenantObserver(deps: TenantObserverDeps): TenantObserver { - const inFlight = new Map>(); - /** `fingerprint` is undefined after a read whose recording failed, so the next read that records bumps the version. */ - const finished = new Map(); - /** A released worker's log never changes once its run ends, so it is read once and kept until the window drops the worker. */ - const endedWorkers = new Map(); - - async function readWorkers(tenantId: string, domain: string): Promise { - if (!deps.workers) return []; - const workers = await deps.workers.deployments(tenantId, new Date(deps.now().getTime() - deps.workers.endedWithinMs)); - for (const [runId, kept] of endedWorkers) { - if (kept.tenantId === tenantId && !workers.some((worker) => worker.runId === runId)) endedWorkers.delete(runId); - } - const observed: ObservedRuns[] = []; - for (const worker of workers) { - const kept = endedWorkers.get(worker.runId); - const runs = kept?.runs ?? (await observeDeployments(deps.observeRuns, [worker], domain))[0]!; - if (!kept && worker.endedAt !== undefined && !isRunning(runs)) endedWorkers.set(worker.runId, { tenantId, runs }); - observed.push(runs); - } - return observed; - } - - async function read(tenantId: string, domain: string): Promise { - try { - const deployments = await deps.liveDeployments(tenantId); - const observed = await observeDeployments(deps.observeRuns, deployments, domain); - const workers = await readWorkers(tenantId, domain); - const runs = mergeObservedRuns([...observed, ...workers]); - const recorded = await deps.recordRuns(tenantId, runs); - const observation = { deployments, observed, workers }; - const seen = fingerprint(runs); - if (finished.get(tenantId)?.fingerprint !== seen) deps.versions.bump(tenantId); - finished.set(tenantId, { at: deps.now().getTime(), observation, ...(recorded && { fingerprint: seen }) }); - return observation; - } finally { - inFlight.delete(tenantId); - } - } - - return { - observe(tenantId, domain) { - const pending = inFlight.get(tenantId); - if (pending) return pending; - const last = finished.get(tenantId); - if (last && deps.now().getTime() - last.at < deps.minIntervalMs) return Promise.resolve(last.observation); - const reading = read(tenantId, domain); - inFlight.set(tenantId, reading); - return reading; - }, - last(tenantId) { - return finished.get(tenantId)?.observation; - }, - }; -} - -export type RecordPassDeps = { - observer: Pick; - /** With `endedSince`, tenants whose child run ended since then count as active: a host's runs may be inserted already completed. */ - activeTenants: (what: "runs" | "deployments", endedSince?: Date) => Promise; - /** How far back an ended run still makes its tenant active; at least a pass interval, so no run's end is missed between passes. */ - endedWithinMs: number; - backoffMs: number; - maxBackoffMs: number; - now: () => Date; - log: (entry: Record) => void; -}; - -export type RecordPass = { - pass: () => Promise<{ retry: boolean }>; - /** False until the tenant's logs were read once since the hub started, if it has a live deployment; until then a run may be going unseen. */ - ready: (tenantId: string) => boolean; -}; - -/** - * The recording loop: each pass reads the logs of every tenant with a run in flight, and of those that had one last pass, so a - * verdict is recorded within a pass of its step completing and a run's end is seen. Every tenant with a live deployment when - * the hub started is read until one read succeeds, so the runs that ended while the hub was down are seen too. - */ -export function createRecordPass(deps: RecordPassDeps): RecordPass { - /** Tenants with a live deployment at start not read yet; undefined until they are listed. */ - let unread: Map | undefined; - let watching = new Map(); - const failing = new Map(); - - /** The tenant is then left until its backoff ends, and the failure logged once per step. */ - async function observe(tenant: ActiveTenant): Promise<{ ok: boolean; running: boolean }> { - const backoff = failing.get(tenant.id); - if (backoff && deps.now().getTime() < backoff.until) return { ok: false, running: true }; - try { - const observation = await deps.observer.observe(tenant.id, tenant.domain); - const running = [...observation.observed, ...observation.workers].some(isRunning); - failing.delete(tenant.id); - return { ok: true, running }; - } catch (err) { - const failures = (backoff?.failures ?? 0) + 1; - const waitMs = Math.min(deps.backoffMs * 2 ** (failures - 1), deps.maxBackoffMs); - failing.set(tenant.id, { failures, until: deps.now().getTime() + waitMs }); - deps.log({ level: "error", msg: "triage_runs_unobserved", tenantId: tenant.id, failures, retryInMs: waitMs, error: String(err) }); - return { ok: false, running: true }; - } - } - - async function pass() { - unread ??= new Map((await deps.activeTenants("deployments")).map((tenant) => [tenant.id, tenant])); - const active = await deps.activeTenants("runs", new Date(deps.now().getTime() - deps.endedWithinMs)); - const tenants = new Map([...unread, ...watching, ...active.map((tenant) => [tenant.id, tenant] as const)]); - const running = new Map(); - for (const tenant of tenants.values()) { - const read = await observe(tenant); - if (read.ok) unread.delete(tenant.id); - if (read.running) running.set(tenant.id, tenant); - } - watching = running; - return { retry: false }; - } - - return { - pass, - ready(tenantId) { - return unread !== undefined && !unread.has(tenantId); - }, - }; -} diff --git a/apps/hub/src/github/triage-reconciler.test.ts b/apps/hub/src/github/triage-reconciler.test.ts index 8031551d9..67437d0ae 100644 --- a/apps/hub/src/github/triage-reconciler.test.ts +++ b/apps/hub/src/github/triage-reconciler.test.ts @@ -1,39 +1,27 @@ import { describe, expect, test } from "bun:test"; -import { createEd25519Crypto, generateKeyPair } from "@intx/crypto"; -import { decodeMail, extractAddrSpec } from "@intx/mime"; import { formatRunAddress } from "@intx/types"; -import { workflowRunRepoIdForAddress, type RepoId, type WorkflowRunEvent, type WorkflowRunReader } from "@intx/hub-sessions"; -import { emptyPack, stockTriggerMail, type ActionKind, type CheckPack, type PrTriageRow } from "@corbits/triage-contracts"; +import { emptyPack, type ActionKind, type CheckPack, type PrTriageRow } from "@corbits/triage-contracts"; import type { CheckRun } from "@corbits/github-reads"; +import { heldForCi, unsettledBy } from "@corbits/triage-workflows/logic/settled"; import { DEFAULT_RECONCILE_POLICY, planTenant, runKey, type ObservedRun, type OpenPr } from "./reconcile-plan.js"; import { triageReconcilePolicy } from "../env.js"; import { TriageStateConflictError, type TriageStateStore } from "./triage-state-store.js"; -import { createRunTriggerDeliverer } from "@corbits/webhooks"; import type { LiveDeployment } from "./deployment.js"; -import { createRunRecorder } from "./run-recorder.js"; -import { createTenantObserver, createTenantVersions } from "./tenant-observer.js"; -import type { ApprovedDo, HostedVerdict, RecordedVerdict } from "./verdict-record-store.js"; +import type { HostedVerdict } from "./verdict-record-store.js"; import { createTriageReconciler, type ReconcileOutcome } from "./triage-reconciler.js"; import { createReconcileLoop } from "./reconcile-loop.js"; import { createRepoActivity, type RepoActivity } from "./repo-activity.js"; import { GithubRateLimited } from "./github-usage.js"; import { markBaseMoved } from "./triage-queue.js"; -import { createTriageRuns, verdictFor } from "./triage-runs.js"; -import type { ToolCall, ToolResult } from "@intx/types/runtime"; -import { evaluate, parts, rules } from "@corbits/triage-workflows/actions/index"; +import { evaluate, rules } from "@corbits/triage-workflows/actions/index"; import { readFacts } from "@corbits/triage-workflows/actions/facts"; -import type { JudgeChunk } from "@corbits/triage-workflows/actions/rules"; -import type { Verdict } from "@corbits/triage-workflows/logic/render"; const REPO = "acme/widgets"; const DOMAIN = "acme.test"; const TENANT_ID = "tnt_acme"; const NOW = new Date("2026-10-07T12:00:00.000Z"); const LONG_AGO = "2026-10-07T00:00:00.000Z"; -const VERSION = "0.1.0-sha-11111111"; -const OFFERING = "off_decision"; -const LIVE = deployment("run_live", new Date(LONG_AGO)); -/** A listener names no workflow version or offering. */ +const STARTED = "2026-10-07T01:00:00.000Z"; const LISTENER = { runId: "run_listener", address: formatRunAddress("run_listener", DOMAIN), createdAt: new Date(LONG_AGO), cancelling: false }; const MINUTE = 60_000; const BASE_MOVE_WINDOW = DEFAULT_RECONCILE_POLICY.baseMoveWindowMs / MINUTE; @@ -41,15 +29,7 @@ const BASE_MOVE_WINDOW = DEFAULT_RECONCILE_POLICY.baseMoveWindowMs / MINUTE; function at(minutes: number): Date { return new Date(NOW.getTime() + minutes * MINUTE); } -let failNextList = false; - -function deployment(runId: string, createdAt: Date): LiveDeployment { - return { runId, address: formatRunAddress(runId, DOMAIN), createdAt, cancelling: false, workflowVersion: VERSION, offeringId: OFFERING }; -} - -function repoKey(anchorRunId: string): string { - return workflowRunRepoIdForAddress(formatRunAddress(anchorRunId, DOMAIN)).id; -} +let failNextRead = false; function head(number: number): OpenPr { return { number, headSha: `sha${number}`, updatedAt: LONG_AGO, draft: false }; @@ -59,34 +39,34 @@ function heads(from: number, count: number): OpenPr[] { return Array.from({ length: count }, (_, i) => head(from + i)); } -function startedBy(payload: unknown, at: string): WorkflowRunEvent { - return { seq: 0, type: "RunStarted", body: { type: "RunStarted", seq: 0, at, trigger: { type: "mail", payload } } }; +function current(number: number): Record { + return { degraded: null, reason: "", headSha: `sha${number}`, checks: [] }; } -function started(number: number, at = "2026-10-07T01:00:00.000Z"): WorkflowRunEvent { - return startedBy(stockTriggerMail({ kind: "pr", repo: REPO, prNumber: number, headSha: `sha${number}` }), at); +/** A manager's run of `number`'s head as the hub reads it: running, or completed with what its verdict settles. */ +function ran(runId: string, number: number, verdict?: Record, startedAt = STARTED): ObservedRun { + if (!verdict) return { runId, source: "github", status: "running", startedAt }; + return { runId, source: "github", status: "completed", startedAt, ...unsettledBy(verdict, `sha${number}`), ...(heldForCi(verdict) && { heldForCi: true }) }; } -function startedBatch(numbers: number[], at = "2026-10-07T01:00:00.000Z"): WorkflowRunEvent { - return startedBy(stockTriggerMail({ kind: "pr", repo: REPO, items: numbers.map((number) => ({ prNumber: number, headSha: `sha${number}` })) }), at); -} +/** The managers' runs by repository, then by head. */ +type Runs = Record>; -function current(number: number): Record { - return { degraded: null, reason: "", headSha: `sha${number}`, checks: [] }; +function runsOf(...observed: Array<[number, ObservedRun]>): Runs { + const byHead: Record = {}; + for (const [number, run] of observed) (byHead[runKey(number, `sha${number}`)] ??= []).push(run); + return { [REPO]: byHead }; } -function replied(stepId: string, reply: Record): WorkflowRunEvent { - const output = { ref: `inline:${JSON.stringify({ reply: JSON.stringify(reply), turn: 1 })}` }; - return { seq: 1, type: "StepCompleted", body: { type: "StepCompleted", seq: 1, at: "2026-10-07T01:30:00.000Z", stepId, output } }; +function settled(from: number, count: number): Runs { + return runsOf(...heads(from, count).map(({ number }): [number, ObservedRun] => [number, ran(`run_${number}`, number, current(number))])); } -function rendered(verdict: Record): WorkflowRunEvent { - return replied("render", verdict); +/** The verdict the hub recorded from `runId`, as the build poll reads it. */ +function recordedOf(runId: string, verdict: { checks: Array<{ check: string; result: string }> }): HostedVerdict { + return { tenantId: TENANT_ID, runId, repo: REPO, source: "github", number: 1, headSha: "sha1", startedAt: new Date(STARTED), verdictAt: new Date(STARTED), settled: true, state: "awaiting-review", actor: "system", humanGated: false, degraded: null, checks: verdict.checks.map(({ check, result }) => ({ check, result })), dos: [], commented: false, detail: { checks: verdict.checks } as HostedVerdict["detail"], trigger: "pull_request:opened", runStatus: null }; } -const completed: WorkflowRunEvent = { seq: 2, type: "RunCompleted", body: { type: "RunCompleted", seq: 2 } }; - -type Logs = Record>; type Delivered = Array<{ payload: Record }>; function mailedHeads(delivered: Delivered): Array<[string, number]> { @@ -97,101 +77,14 @@ function mailedNumbers(delivered: Delivered): number[] { return mailedHeads(delivered).map(([, number]) => number); } -function githubAnswer({ id, name }: ToolCall, pr: Record): Record { - const n = Number(id.split(":")[1]); - switch (name) { - case "github_list_open_prs": - return { prs: [1, 2, 3].map((number) => ({ number, title: `PR ${number}` })) }; - case "github_read_repo_docs": - return { docs: [], rules: [], ref: "tree" }; - case "github_get_pr": - return { title: `PR ${n}`, author: "octocat", sha: `sha${n}`, state: "open", draft: false, requestedReviewers: 0, ...pr }; - case "github_list_pr_files": - return { files: [{ path: `src/pr${n}.ts`, status: "modified", patch: `@@ -0,0 +1 @@\n+export const pr${n} = ${n};` }] }; - case "github_get_reviews": - return { reviews: [] }; - case "github_list_pr_commits": - return { commits: [] }; - case "github_list_issue_comments": - return { comments: [] }; - case "github_get_checks": - return { checks: [] }; - case "github_get_stack": - return { includes: [] }; - } - throw new Error(`unexpected call ${name}`); -} - -function focusedAnswers(questions: Array<{ id: string }>, related: number): string { - const rest = (1 - related) / 2; - const probabilities: Record> = { - focused: { one_thing: related, more_than_one_thing: rest, cant_tell: rest }, - verdict: { pass: related, fail: rest, cant_tell: rest }, - }; - return questions.map(({ id }) => JSON.stringify(id === "instructed" ? { id, type: "noul", noul: 0.1 } : { id, type: "choice", choice: Object.keys(probabilities[id]!)[0], probabilities: probabilities[id], confidence: 0.9 })).join(""); -} - -async function triageMail(mail: unknown, pr: Record, related = 0.8) { - async function github(asked: ToolCall): Promise { - return { callId: asked.id, content: githubAnswer(asked, pr) }; - } - const read = await readFacts(mail, () => github); - const ctx = {} as never; - const signal = new AbortController().signal; - const ruled = await rules(read, ctx, signal); - function answer(chunks: JudgeChunk[]) { - return chunks.map((chunk) => ({ reply: JSON.stringify({ judge: focusedAnswers(chunk.questions, related) }) })); - } - const judged = { ...ruled, judge: { output: answer(ruled.chunks) } }; - const { chunks } = await parts(judged, ctx, signal); - const verdicts = await evaluate(ruled.needsJudgment ? { ...judged, judgeParts: { output: answer(chunks) } } : ruled, ctx, signal) as { items: Verdict[] }; - return { asked: ruled.chunks.length, verdicts: verdicts.items }; -} - -function evaluated(output: unknown): WorkflowRunEvent { - return { seq: 1, type: "StepCompleted", body: { type: "StepCompleted", seq: 1, at: "2026-10-07T01:30:00.000Z", stepId: "evaluate", output: { ref: `inline:${JSON.stringify(output)}` } } }; -} - -function fakeReader(logs: Logs) { - const reads: Array<{ repo: string; runId: string }> = []; - const latestOf: string[][] = []; - const reader: WorkflowRunReader = { - async listRunIds(repoId: RepoId) { - return Object.keys(logs[repoId.id] ?? {}); - }, - async readRunEvents(repoId: RepoId, _ref: string, runId: string) { - reads.push({ repo: repoId.id, runId }); - return logs[repoId.id]?.[runId] ?? []; - }, - async readLatestRunEvents(repoId: RepoId, _ref: string, include: (runId: string) => boolean) { - const events = new Map(); - for (const [runId, log] of Object.entries(logs[repoId.id] ?? {})) { - if (include(runId)) events.set(runId, log.at(-1) ?? null); - } - latestOf.push([repoId.id, ...events.keys()]); - return { tip: "tip", events }; - }, - async resolveRefTip() { - return "tip"; - }, - async hasRepository() { - return true; - }, - }; - return { reader, reads, latestOf }; -} - type HarnessOptions = { github?: boolean; maxInFlight?: string; refuseSaves?: number; onRefusedSave?: (repo: string) => void; now?: () => Date; - deployments?: LiveDeployment[]; /** The host's listener; a live one when not given. */ listener?: LiveDeployment | undefined; - /** The managers' runs by head, read on each pass. */ - managed?: () => Record; pack?: CheckPack; automatic?: ActionKind[]; noPack?: boolean; @@ -201,7 +94,7 @@ type HarnessOptions = { headChecks?: CheckRun[]; /** Stored triage state of the repository before the first pass. */ rows?: PrTriageRow[]; - /** Verdicts recorded from runs no live log holds any more. */ + /** Verdicts the hub recorded. */ recorded?: HostedVerdict[]; activity?: RepoActivity; /** Thrown by every listing of open pull requests while set. */ @@ -210,47 +103,19 @@ type HarnessOptions = { sendErrors?: Record; }; -function harness(logs: Logs, prs: OpenPr[] | Record, options: HarnessOptions = {}) { +/** `runs` is what the managers' runs are on each pass. */ +function harness(runs: Runs | (() => Runs), prs: OpenPr[] | Record, options: HarnessOptions = {}) { const repos = Array.isArray(prs) ? { [REPO]: prs } : prs; function tenant() { const ns = { repos: Object.keys(repos).map((name) => ({ name, connected: true, enabled: true })) }; return { id: TENANT_ID, domain: DOMAIN, config: { corbitsTriage: ns } }; } - const { reader, reads, latestOf } = fakeReader(logs); const delivered: Delivered = []; const saved = new Map(options.rows ? [[REPO, options.rows]] : []); let refusals = options.refuseSaves ?? 0; const logged: Array> = []; const listings: string[] = []; const now = options.now ?? (() => NOW); - const deployments = (options.deployments ?? [LIVE]).map((live) => ({ ...live })); - const verdicts: RecordedVerdict[] = []; - const approved: ApprovedDo[] = []; - async function liveDeployments() { - if (failNextList) { - failNextList = false; - throw new Error("transient list failure"); - } - return deployments.map((live) => ({ ...live })); - } - const observer = createTenantObserver({ - liveDeployments, - observeRuns: createTriageRuns({ runReader: reader, readSettled: async () => new Map(), maxKnownRuns: 100 }), - recordRuns: createRunRecorder({ - verdicts: { - async record(_tenantId, recorded) { - verdicts.push(...recorded); - }, - async recordApproved(_tenantId, outcomes) { - approved.push(...outcomes); - }, - }, - log: () => {}, - }), - versions: createTenantVersions(), - minIntervalMs: 0, - now, - }); const reconcile = createTriageReconciler({ source: "github", tenants: async function tenants() { @@ -266,8 +131,14 @@ function harness(logs: Logs, prs: OpenPr[] | Record, options: }; }, activity: options.activity ?? { stale: () => true }, - observeTenant: observer.observe, - managerRuns: async () => ({ byRepo: new Map(options.managed ? [[REPO, new Map(Object.entries(options.managed()))]] : []), runCount: 0, verdicts: [], approved: [] }), + managerRuns: async function managerRuns() { + if (failNextRead) { + failNextRead = false; + throw new Error("transient read failure"); + } + const byRepo = typeof runs === "function" ? runs() : runs; + return { byRepo: new Map(Object.entries(byRepo).map(([repo, byHead]) => [repo, new Map(Object.entries(byHead))])) }; + }, headChecks: async () => options.headChecks ?? [], verdicts: { latest: async () => options.recorded ?? [] }, store: store(), @@ -301,13 +172,7 @@ function harness(logs: Logs, prs: OpenPr[] | Record, options: }, }; } - return { reconcile, listings, delivered, saved, logged, reads, latestOf, deployments, verdicts, approved, store: store() }; -} - -function settled(from: number, count: number): Record { - const runs: Record = {}; - for (const { number } of heads(from, count)) runs[`run_${number}`] = [started(number), rendered(current(number)), completed]; - return runs; + return { reconcile, listings, delivered, saved, logged, store: store() }; } describe("triage reconciler", () => { @@ -318,44 +183,45 @@ describe("triage reconciler", () => { expect(run.logged).toContainEqual(expect.objectContaining({ msg: "triage_reconcile_skipped", reason: "no_listener" })); }); - test("a host whose listener is live is caught up with no pr-triage deployment", async () => { - const run = harness({}, [head(1)], { deployments: [], listener: LISTENER }); + test("a host whose listener is live is caught up", async () => { + const run = harness({}, [head(1)], { listener: LISTENER }); await run.reconcile(); expect(mailedNumbers(run.delivered)).toEqual([1]); }); test("a head mailed through the listener is mailed once while its manager runs and once it reported", async () => { let minutes = 0; - let managed: Record = {}; - const run = harness({}, [head(1)], { deployments: [], listener: LISTENER, now: () => at(minutes), managed: () => managed }); + let managed: Runs = {}; + const run = harness(() => managed, [head(1)], { listener: LISTENER, now: () => at(minutes) }); await run.reconcile(); - managed = { [runKey(1, "sha1")]: [{ runId: "run_manager", source: "github", status: "running", startedAt: at(1).toISOString() }] }; + managed = runsOf([1, ran("run_manager", 1, undefined, at(1).toISOString())]); for (minutes = 1; minutes <= 3; minutes++) await run.reconcile(); - managed = { [runKey(1, "sha1")]: [{ runId: "run_manager", source: "github", status: "completed", startedAt: at(1).toISOString() }] }; + managed = runsOf([1, ran("run_manager", 1, current(1), at(1).toISOString())]); for (minutes = 4; minutes <= 60; minutes += 14) await run.reconcile(); expect(mailedNumbers(run.delivered)).toEqual([1]); }); test("a host without build webhooks: a head whose verdict waits on CI is queued again once its builds stop pending", async () => { const waiting = { ...current(1), repo: REPO, number: 1, state: "awaiting-review", actor: "system", humanGated: false, checks: [{ check: "ci-green", kind: "machine", result: "unconfirmed" }] }; - const logs: Logs = { [repoKey(LIVE.runId)]: { run_1: [started(1), rendered(waiting), completed] } }; - const pending = harness(logs, [head(1)], { headChecks: [{ name: "build", status: "in_progress", conclusion: null }] }); + const logs = runsOf([1, ran("run_1", 1, waiting)]); + const recorded = [recordedOf("run_1", waiting)]; + const pending = harness(logs, [head(1)], { recorded, headChecks: [{ name: "build", status: "in_progress", conclusion: null }] }); await pending.reconcile(); expect(pending.delivered).toHaveLength(0); - const finished = harness(logs, [head(1)], { headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); + const finished = harness(logs, [head(1)], { recorded, headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); await finished.reconcile(); expect(mailedNumbers(finished.delivered)).toEqual([1]); expect(finished.logged).toContainEqual(expect.objectContaining({ msg: "triage_requeued", pr: 1, reason: "build_status_changed" })); // A verdict that failed on CI is polled too: red turning green is caught without another event. const failed = { ...waiting, checks: [{ check: "ci-green", kind: "machine", result: "fail" }] }; - const red = { [repoKey(LIVE.runId)]: { run_1: [started(1), rendered(failed), completed] } }; - const stillRed = harness(red, [head(1)], { headChecks: [{ name: "build", status: "completed", conclusion: "failure" }] }); + const red = runsOf([1, ran("run_1", 1, failed)]); + const stillRed = harness(red, [head(1)], { recorded: [recordedOf("run_1", failed)], headChecks: [{ name: "build", status: "completed", conclusion: "failure" }] }); await stillRed.reconcile(); expect(stillRed.delivered).toHaveLength(0); - const green = harness(red, [head(1)], { headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); + const green = harness(red, [head(1)], { recorded: [recordedOf("run_1", failed)], headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); await green.reconcile(); expect(mailedNumbers(green.delivered)).toEqual([1]); - const nowCancelled = harness(logs, [head(1)], { headChecks: [{ name: "build", status: "completed", conclusion: "cancelled" }] }); + const nowCancelled = harness(logs, [head(1)], { recorded, headChecks: [{ name: "build", status: "completed", conclusion: "cancelled" }] }); await nowCancelled.reconcile(); expect(mailedNumbers(nowCancelled.delivered)).toEqual([1]); }); @@ -363,8 +229,8 @@ describe("triage reconciler", () => { test("a verdict that read CI as cancelled is not queued again while it stays cancelled, however many reads, and is once a check is re-run", async () => { const cancelled = { ...current(1), repo: REPO, number: 1, state: "awaiting-review", actor: "system", humanGated: false, checks: [{ check: "ci-green", kind: "machine", result: "unconfirmed", output: { kind: "values", values: { status: "cancelled" } } }] }; let now = NOW; - const options: HarnessOptions = { now: () => now, headChecks: [{ name: "build", status: "completed", conclusion: "cancelled" }] }; - const run = harness({ [repoKey(LIVE.runId)]: { run_1: [started(1), rendered(cancelled), completed] } }, [head(1)], options); + const options: HarnessOptions = { now: () => now, recorded: [recordedOf("run_1", cancelled)], headChecks: [{ name: "build", status: "completed", conclusion: "cancelled" }] }; + const run = harness(runsOf([1, ran("run_1", 1, cancelled)]), [head(1)], options); for (let pass = 0; pass <= DEFAULT_RECONCILE_POLICY.maxAttempts; pass++) { now = new Date(now.getTime() + DEFAULT_RECONCILE_POLICY.backoffMaxMs); await run.reconcile(); @@ -376,15 +242,15 @@ describe("triage reconciler", () => { expect(mailedNumbers(run.delivered)).toEqual([1]); }); - test("a head whose verdict waits on CI is queued again once CI finished, though the deployment that gave the verdict was replaced", async () => { + test("a head whose recorded verdict waits on CI is queued again once CI finished", async () => { const triaged: PrTriageRow = { number: 1, headSha: "sha1", status: "triaged", attempts: 1, runId: "run_gone", firstSeenAt: LONG_AGO, updatedAt: LONG_AGO }; const recorded: HostedVerdict = { tenantId: TENANT_ID, runId: "run_gone", repo: REPO, source: "github", number: 1, headSha: "sha1", startedAt: new Date(LONG_AGO), verdictAt: new Date(LONG_AGO), settled: true, state: "awaiting-review", actor: "system", humanGated: false, degraded: null, checks: [{ check: "ci-green", result: "unconfirmed" }], dos: [], commented: false, detail: null, trigger: "pull_request:opened", runStatus: null }; - const run = harness({ [repoKey(LIVE.runId)]: {} }, [head(1)], { rows: [triaged], recorded: [recorded], headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); + const run = harness({}, [head(1)], { rows: [triaged], recorded: [recorded], headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); await run.reconcile(); expect(mailedNumbers(run.delivered)).toEqual([1]); expect(run.logged).toContainEqual(expect.objectContaining({ msg: "triage_requeued", pr: 1, reason: "build_status_changed" })); // Without the rule's detail, cancelled CI is taken as already read, so the head is not run again and again. - const cancelled = harness({ [repoKey(LIVE.runId)]: {} }, [head(1)], { rows: [triaged], recorded: [recorded], headChecks: [{ name: "build", status: "completed", conclusion: "cancelled" }] }); + const cancelled = harness({}, [head(1)], { rows: [triaged], recorded: [recorded], headChecks: [{ name: "build", status: "completed", conclusion: "cancelled" }] }); await cancelled.reconcile(); expect(cancelled.delivered).toHaveLength(0); }); @@ -392,26 +258,26 @@ describe("triage reconciler", () => { test("a head waiting on CI under another host's verdict of the same repository, number and head is not queued again", async () => { const triaged: PrTriageRow = { number: 1, headSha: "sha1", status: "triaged", attempts: 1, runId: "run_gone", firstSeenAt: LONG_AGO, updatedAt: LONG_AGO }; const recorded: HostedVerdict = { tenantId: TENANT_ID, runId: "run_bitbucket", repo: REPO, source: "bitbucket-datacenter", number: 1, headSha: "sha1", startedAt: new Date(LONG_AGO), verdictAt: new Date(LONG_AGO), settled: true, state: "awaiting-review", actor: "system", humanGated: false, degraded: null, checks: [{ check: "ci-green", result: "unconfirmed" }], dos: [], commented: false, detail: null, trigger: "pull_request:opened", runStatus: null }; - const run = harness({ [repoKey(LIVE.runId)]: {} }, [head(1)], { rows: [triaged], recorded: [recorded], headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); + const run = harness({}, [head(1)], { rows: [triaged], recorded: [recorded], headChecks: [{ name: "build", status: "completed", conclusion: "success" }] }); await run.reconcile(); expect(run.delivered).toHaveLength(0); }); test("a head the reconciler holds for CI is mailed to be marked waiting once, where triage writes labels", async () => { const pack = { ...emptyPack(REPO), checks: { "ci-green": { enabled: true } } }; - const run = harness({ [repoKey(LIVE.runId)]: {} }, [head(1)], { pack, automatic: ["labels"], headChecks: [{ name: "build", status: "in_progress", conclusion: null }] }); + const run = harness({}, [head(1)], { pack, automatic: ["labels"], headChecks: [{ name: "build", status: "in_progress", conclusion: null }] }); await run.reconcile(); await run.reconcile(); expect(run.delivered.map((entry) => entry.payload)).toEqual([expect.objectContaining({ kind: "waiting", prNumber: 1, headSha: "sha1" })]); }); test("a repository with every rule off, or no policy, starts no run and is logged once per head", async () => { - const off = harness({ [repoKey(LIVE.runId)]: {} }, [head(1), head(2)], { pack: emptyPack(REPO) }); + const off = harness({}, [head(1), head(2)], { pack: emptyPack(REPO) }); await off.reconcile(); expect(off.delivered).toHaveLength(0); await off.reconcile(); expect(off.logged.filter((entry) => entry.msg === "triage_skipped_no_rules").map((entry) => entry.pulls)).toEqual([[1, 2]]); - const unset = harness({ [repoKey(LIVE.runId)]: {} }, [head(1)], { noPack: true }); + const unset = harness({}, [head(1)], { noPack: true }); await unset.reconcile(); await unset.reconcile(); expect(unset.delivered).toHaveLength(0); @@ -419,97 +285,20 @@ describe("triage reconciler", () => { }); test("a policy triage can't read mails each open head with why and no pack, so the run marks it failed", async () => { - const run = harness({ [repoKey(LIVE.runId)]: {} }, [head(1)], { unreadable: "The policy file has unknown checks: max-size.", automatic: ["labels"] }); + const run = harness({}, [head(1)], { unreadable: "The policy file has unknown checks: max-size.", automatic: ["labels"] }); await run.reconcile(); expect(run.delivered.map((entry) => entry.payload)).toEqual([expect.objectContaining({ policyUnreadable: "The policy file has unknown checks: max-size.", automatic: ["labels"], items: [{ prNumber: 1, headSha: "sha1" }] })]); expect(run.delivered[0]!.payload).not.toHaveProperty("pack"); }); - test("reads only the live deployment's runs and queues to it", async () => { - const logs: Logs = { - [repoKey("run_old")]: { run_old_pr: [started(1), rendered(current(1)), completed] }, - [repoKey(LIVE.runId)]: {}, - }; - const { reconcile, delivered, reads, latestOf } = harness(logs, [head(1)]); - await reconcile(); - expect(latestOf.map(([repo]) => repo)).toEqual([repoKey(LIVE.runId)]); - expect(reads.every((read) => read.repo === repoKey(LIVE.runId))).toBe(true); - expect(delivered).toHaveLength(1); - expect(delivered[0]).toMatchObject({ payload: { kind: "pr", repo: REPO, items: [{ prNumber: 1, headSha: "sha1" }] } }); - }); - - test("the Dos a run executed are recorded on its verdict", async () => { - const comment = { id: "hello", branch: "always", index: 0, kind: "comment", reason: "", effectId: "e-comment", target: { body: "hi" } }; - const label = { ...comment, index: 1, kind: "labels", effectId: "e-label", target: { labels: ["x"] } }; - const verdict = { ...current(1), repo: REPO, number: 1, state: "ready", actor: "maintainer", humanGated: false, actions: [comment, label] }; - const executed = replied("execute", { dos: [{ effectId: "e-comment", status: "done" }, { effectId: "e-label", status: "failed", error: "github POST -> 500" }] }); - const log = [started(1), rendered(verdict), executed, completed]; - const { reconcile, verdicts } = harness({ [repoKey(LIVE.runId)]: { run_1: log } }, [head(1)]); - await reconcile(); - const outcomes = [{ effectId: "e-comment", status: "done" }, { effectId: "e-label", status: "failed", error: "github POST -> 500" }]; - expect(verdictFor(log, REPO, 1)?.["actions"]).toMatchObject(outcomes); - expect(verdicts.at(-1)?.detail?.actions).toMatchObject(outcomes); - }); - - test("a held Do's approval run puts its outcome on the verdicts of its head, once", async () => { - const held = { id: "hello", branch: "always", index: 0, kind: "comment", reason: "", effectId: "e-comment", target: { body: "hi" }, held: true }; - const verdict = { ...current(1), repo: REPO, number: 1, state: "ready", actor: "maintainer", humanGated: false, actions: [held] }; - const mail = stockTriggerMail({ kind: "approve-do", runId: "run_1", request: { repo: REPO, number: 1 } }); - const approval = [startedBy(mail, "2026-10-07T02:00:00.000Z"), replied("executeApproved", { dos: [{ effectId: "e-comment", status: "done" }] }), completed]; - const retriaged = [started(1, "2026-10-07T03:00:00.000Z"), rendered(verdict), completed]; - const { reconcile, verdicts, approved } = harness({ [repoKey(LIVE.runId)]: { run_1: [started(1), rendered(verdict), completed], run_2: approval, run_3: retriaged } }, [head(1)]); - await reconcile(); - await reconcile(); - const latest = new Map(verdicts.map((recorded) => [recorded.runId, recorded])); - expect([...latest.keys()].sort()).toEqual(["run_1", "run_3"]); - for (const recorded of latest.values()) expect(recorded.detail?.actions).toMatchObject([{ effectId: "e-comment", held: true, status: "done" }]); - expect(approved).toEqual([{ repo: REPO, number: 1, effectId: "e-comment", status: "done" }]); - }); - - test("a verdict is recorded with what the portal shows of it: its own fields, the facts it read and the webhook event that started it", async () => { - const mail = stockTriggerMail({ kind: "pr", repo: REPO, prNumber: 1, headSha: "sha1", event: "pull_request", action: "synchronize" }); - const facts = { repo: REPO, number: 1, title: "Add widgets", author: "ann", draft: false, mergeable: true, checks: "success", body: "not shown" }; - const ruled: WorkflowRunEvent = { seq: 1, type: "StepCompleted", body: { type: "StepCompleted", seq: 1, stepId: "rules", output: { ref: `inline:${JSON.stringify({ items: [{ facts, det: {} }] })}` } } }; - const check = { check: "ci-green", kind: "machine", result: "pass", reason: "green", evidence: [], gate: false }; - const verdict = { ...current(1), repo: REPO, number: 1, state: "ready-monitoring", actor: "author", humanGated: false, priority: "P2", feedback: "Thanks.", checks: [check], det: { sources: {} } }; - const { reconcile, verdicts } = harness({ [repoKey(LIVE.runId)]: { run_1: [startedBy(mail, "2026-10-07T01:00:00.000Z"), ruled, rendered(verdict), completed] } }, [head(1)]); - await reconcile(); - expect(verdicts).toMatchObject([{ - runId: "run_1", - trigger: "pull_request:synchronize", - detail: { priority: "P2", feedback: "Thanks.", reason: "", checks: [check], facts: { title: "Add widgets", author: "ann", draft: false, mergeable: true, checks: "success" } }, - }]); - expect(verdicts[0]!.detail).not.toHaveProperty("det"); - expect(verdicts[0]!.detail?.facts).not.toHaveProperty("body"); - }); - - test("a verdict is recorded once its step completes, and kept when the run then fails", async () => { - const verdict = { ...current(1), repo: REPO, number: 1, state: "ready-monitoring", actor: "author", humanGated: false }; - const log = [started(1), rendered(verdict)]; - const { reconcile, verdicts } = harness({ [repoKey(LIVE.runId)]: { run_1: log } }, [head(1)]); - await reconcile(); - expect(verdicts.map(({ runId, state, settled, runStatus }) => ({ runId, state, settled, runStatus }))).toEqual([{ runId: "run_1", state: "ready-monitoring", settled: true, runStatus: "running" }]); - log.push({ seq: 3, type: "RunFailed", body: { type: "RunFailed", seq: 3 } }); - await reconcile(); - expect(verdicts.map(({ runId, state, runStatus }) => ({ runId, state, runStatus }))).toEqual([ - { runId: "run_1", state: "ready-monitoring", runStatus: "running" }, - { runId: "run_1", state: "ready-monitoring", runStatus: "failed" }, - ]); - }); - test("draft heads are not queued", async () => { - const { reconcile, delivered } = harness({ [repoKey(LIVE.runId)]: {} }, [{ ...head(1), draft: true }, head(2)]); + const { reconcile, delivered } = harness({}, [{ ...head(1), draft: true }, head(2)]); await reconcile(); expect(mailedNumbers(delivered)).toEqual([2]); }); test("a degraded verdict is queued again and a clean one settles the head", async () => { - const logs: Logs = { - [repoKey(LIVE.runId)]: { - run_degraded: [started(2), rendered({ degraded: "error", reason: "This repository still needs a policy." }), completed], - run_clean: [started(3), rendered(current(3)), completed], - }, - }; + const logs = runsOf([2, ran("run_degraded", 2, { degraded: "error", reason: "This repository still needs a policy." })], [3, ran("run_clean", 3, current(3))]); const { reconcile, delivered, saved } = harness(logs, [head(2), head(3)]); await reconcile(); expect(mailedNumbers(delivered)).toEqual([2]); @@ -520,22 +309,20 @@ describe("triage reconciler", () => { test("verdicts on another commit or with unconfirmed checks are queued again with their reason", async () => { const unconfirmed = [{ check: "no-merge-conflicts", kind: "machine", result: "unconfirmed" }, { check: "docs", kind: "model", result: "unconfirmed" }]; - const logs: Logs = { - [repoKey(LIVE.runId)]: { - run_other_head: [started(2), rendered({ ...current(2), headSha: "sha2-pushed" }), completed], - run_unconfirmed: [started(3), rendered({ ...current(3), checks: unconfirmed }), completed], - run_not_asked: [started(4), rendered({ ...current(4), checks: [{ check: "docs", kind: "model", result: "unconfirmed" }] }), completed], - run_ci_pending: [started(5), rendered({ ...current(5), checks: [{ check: "ci-green", kind: "machine", result: "unconfirmed" }, { check: "docs", kind: "model", result: "pass" }] }), completed], - }, - }; + const logs = runsOf( + [2, ran("run_other_head", 2, { ...current(2), headSha: "sha2-pushed" })], + [3, ran("run_unconfirmed", 3, { ...current(3), checks: unconfirmed })], + [4, ran("run_not_asked", 4, { ...current(4), checks: [{ check: "docs", kind: "model", result: "unconfirmed" }] })], + [5, ran("run_ci_pending", 5, { ...current(5), checks: [{ check: "ci-green", kind: "machine", result: "unconfirmed" }, { check: "docs", kind: "model", result: "pass" }] })], + ); const { reconcile, delivered, saved, logged } = harness(logs, [head(2), head(3), head(4), head(5)]); await reconcile(); expect(mailedNumbers(delivered)).toEqual([2, 3]); const reasons = logged.filter((entry) => entry["msg"] === "triage_requeued").map((entry) => entry["reason"]); expect(reasons).toEqual(["verdict made on head sha2-pushed", "unconfirmed checks: no-merge-conflicts"]); const rows = saved.get(REPO) ?? []; - expect(rows.filter((row) => row.number < 4).every((row) => row.status === "queued" && row.attempts === 1 && row.workflowVersion === VERSION)).toBe(true); - expect(rows.find((row) => row.number === 4)).toMatchObject({ status: "triaged", runId: "run_not_asked", workflowVersion: VERSION }); + expect(rows.filter((row) => row.number < 4).every((row) => row.status === "queued" && row.attempts === 1)).toBe(true); + expect(rows.find((row) => row.number === 4)).toMatchObject({ status: "triaged", runId: "run_not_asked" }); // Running checks settle; the completed check_run webhook triages the head again. expect(rows.find((row) => row.number === 5)).toMatchObject({ status: "triaged", runId: "run_ci_pending" }); }); @@ -549,7 +336,7 @@ describe("triage reconciler", () => { const signal = new AbortController().signal; const verdict = await evaluate(await rules(read, ctx, signal), ctx, signal) as Record; expect(verdict).toMatchObject({ degraded: "error", headSha: null, reason: "facts: pr input has no prNumber" }); - const logs: Logs = { [repoKey(LIVE.runId)]: { run_fail: [started(1), rendered(verdict), completed] } }; + const logs = runsOf([1, ran("run_fail", 1, verdict)]); const { reconcile, delivered, logged } = harness(logs, [head(1)]); await reconcile(); expect(mailedNumbers(delivered)).toEqual([1]); @@ -558,13 +345,10 @@ describe("triage reconciler", () => { test("a merge triages the repository's other settled heads again, not the merged one or another repository's", async () => { const OTHER = "acme/gadgets"; - const runs: Record = { - ...settled(1, 3), - run_other: [startedBy(stockTriggerMail({ kind: "pr", repo: OTHER, prNumber: 4, headSha: "sha4" }), "2026-10-07T01:00:00.000Z"), rendered(current(4)), completed], - }; + const runs: Runs = { ...settled(1, 3), [OTHER]: { [runKey(4, "sha4")]: [ran("run_other", 4, current(4))] } }; let now = NOW; const open = { [REPO]: heads(1, 3), [OTHER]: [head(4)] }; - const { reconcile, delivered, saved, logged, store } = harness({ [repoKey(LIVE.runId)]: runs }, open, { now: () => now }); + const { reconcile, delivered, saved, logged, store } = harness(() => runs, open, { now: () => now }); await reconcile(); expect(delivered).toEqual([]); @@ -577,16 +361,18 @@ describe("triage reconciler", () => { expect(logged.filter((entry) => entry["msg"] === "triage_requeued").map((entry) => entry["reason"])).toEqual(["base branch moved", "base branch moved"]); expect(saved.get(OTHER)?.[0]).toMatchObject({ status: "triaged" }); - runs["run_recheck"] = [startedBatch([1, 3], at(2).toISOString()), rendered({ items: [current(1), current(3)] }), completed]; + for (const number of [1, 3]) runs[REPO]![runKey(number, `sha${number}`)]!.push(ran("run_recheck", number, current(number), at(2).toISOString())); now = at(3); await reconcile(); expect(saved.get(REPO)?.map((row) => [row.number, row.status, row.runId])).toEqual([[1, "triaged", "run_recheck"], [3, "triaged", "run_recheck"]]); }); test("a head in flight when the base branch moves runs again once its run ends and its base move window has passed", async () => { - const runs: Record = { run_1: [started(1, at(-1).toISOString())], run_3: [started(3), rendered(current(3)), completed] }; + const runs = runsOf([1, ran("run_1", 1, undefined, at(-1).toISOString())], [3, ran("run_3", 3, current(3))]); + const of1 = runs[REPO]![runKey(1, "sha1")]!; + const of3 = runs[REPO]![runKey(3, "sha3")]!; let now = NOW; - const { reconcile, delivered, saved, store } = harness({ [repoKey(LIVE.runId)]: runs }, [head(1), head(3)], { now: () => now }); + const { reconcile, delivered, saved, store } = harness(runs, [head(1), head(3)], { now: () => now }); await reconcile(); expect(saved.get(REPO)?.map((row) => row.status)).toEqual(["running", "triaged"]); @@ -598,7 +384,7 @@ describe("triage reconciler", () => { expect(saved.get(REPO)?.map((row) => row.status)).toEqual(["running", "queued"]); // #3's run starts before a second merge and ends after it; #1's run outlives both. - runs["run_3b"] = [started(3, at(2.5).toISOString())]; + of3.push(ran("run_3b", 3, undefined, at(2.5).toISOString())); now = at(4); expect(await markBaseMoved(store, TENANT_ID, REPO, 8, now, DEFAULT_RECONCILE_POLICY)).toBe(0); now = at(5); @@ -606,14 +392,14 @@ describe("triage reconciler", () => { expect(mailedNumbers(delivered)).toEqual([3]); expect(saved.get(REPO)?.map((row) => row.status)).toEqual(["running", "running"]); - runs["run_1"]!.push(rendered(current(1)), completed); - runs["run_3b"]!.push(rendered(current(3)), completed); + of1.splice(0, 1, ran("run_1", 1, current(1), at(-1).toISOString())); + of3.splice(1, 1, ran("run_3b", 3, current(3), at(2.5).toISOString())); now = at(6); await reconcile(); // #1 was never queued by the hub, so it goes at once; #3, queued at minute 2, waits out its window. expect(mailedNumbers(delivered)).toEqual([3, 1]); - runs["run_1b"] = [started(1, at(6).toISOString()), rendered(current(1)), completed]; + of1.push(ran("run_1b", 1, current(1), at(6).toISOString())); now = at(1 + BASE_MOVE_WINDOW); await reconcile(); expect(mailedNumbers(delivered)).toEqual([3, 1]); @@ -621,7 +407,7 @@ describe("triage reconciler", () => { await reconcile(); expect(mailedNumbers(delivered)).toEqual([3, 1, 3]); - runs["run_3c"] = [started(3, now.toISOString()), rendered(current(3)), completed]; + of3.push(ran("run_3c", 3, current(3), now.toISOString())); now = at(3 + BASE_MOVE_WINDOW); await reconcile(); expect(saved.get(REPO)?.map((row) => [row.number, row.status, row.runId])).toEqual([[1, "triaged", "run_1b"], [3, "triaged", "run_3c"]]); @@ -645,7 +431,7 @@ describe("triage reconciler", () => { test("merges inside a head's base move window re-check it once when the window closes, not once per merge", async () => { const runs = settled(1, 2); let now = NOW; - const { reconcile, delivered, store } = harness({ [repoKey(LIVE.runId)]: runs }, heads(1, 2), { now: () => now }); + const { reconcile, delivered, store } = harness(runs, heads(1, 2), { now: () => now }); await reconcile(); now = at(1); await markBaseMoved(store, TENANT_ID, REPO, 9, now, DEFAULT_RECONCILE_POLICY); @@ -653,7 +439,7 @@ describe("triage reconciler", () => { await reconcile(); expect(mailedNumbers(delivered)).toEqual([1, 2]); - runs["run_recheck"] = [startedBatch([1, 2], at(2).toISOString()), rendered({ items: [current(1), current(2)] }), completed]; + for (const number of [1, 2]) runs[REPO]![runKey(number, `sha${number}`)]!.push(ran("run_recheck", number, current(number), at(2).toISOString())); for (const minute of [4, 6, 8, 10, 12]) { now = at(minute); await markBaseMoved(store, TENANT_ID, REPO, minute, now, DEFAULT_RECONCILE_POLICY); @@ -673,100 +459,10 @@ describe("triage reconciler", () => { const row: PrTriageRow = { number: 7, headSha: "abc", status: "queued", attempts: 1, firstSeenAt: LONG_AGO, queuedAt: iso(moved + 50), staleBefore: iso(moved), updatedAt: iso(moved + 50) }; const run: ObservedRun = { runId: "run_after", source: "github", status: "completed", startedAt: iso(moved + 50 - 500) }; const pr = { number: 7, headSha: "abc", updatedAt: LONG_AGO, draft: false }; - const plans = planTenant({ repos: [{ name: REPO, prs: [pr], rows: [row] }], runs: new Map([[REPO, new Map([["7@abc", [run]]])]]), now: at(1), policy: DEFAULT_RECONCILE_POLICY, workflowVersion: undefined }); + const plans = planTenant({ repos: [{ name: REPO, prs: [pr], rows: [row] }], runs: new Map([[REPO, new Map([["7@abc", [run]]])]]), now: at(1), policy: DEFAULT_RECONCILE_POLICY }); expect(plans.get(REPO)!.rows[0]!.status).toBe("triaged"); }); - describe("judge answers", () => { - const pack: CheckPack = { ...emptyPack(REPO), checks: { "no-merge-conflicts": { enabled: true }, focused: { enabled: true } } }; - const policy = { enabled: true, checkPack: { name: REPO } }; - - async function remailed(related: number, listener = LIVE) { - const first = await triageMail(stockTriggerMail({ kind: "pr", repo: REPO, items: [1, 3].map((n) => ({ prNumber: n, headSha: `sha${n}` })), policy, checkPack: pack }), { mergeable: true }, related); - const runs = { run_first: [startedBatch([1, 3]), evaluated({ items: first.verdicts }), completed] }; - const { reconcile, delivered, store } = harness({ [repoKey(LIVE.runId)]: runs }, [head(1), head(3)], { pack, listener }); - await reconcile(); - await markBaseMoved(store, TENANT_ID, REPO, 2, NOW, DEFAULT_RECONCILE_POLICY); - await reconcile(); - return { first, payload: delivered[0]!.payload }; - } - - test("a head re-checked after a merge reuses its decisive answers, while its conflicts are read afresh", async () => { - const { first, payload } = await remailed(0.8); - expect(first.asked).toBe(2); - expect((payload["items"] as Array<{ judged?: object }>).map((item) => Object.keys(item.judged ?? {}).length)).toEqual([2]); - const reused = await triageMail(stockTriggerMail(payload), { mergeable: true }); - expect(reused.asked).toBe(0); - expect(reused.verdicts.map(({ verdict, checks }) => [verdict, checks.find((check) => check.check === "focused")?.result])).toEqual([["pass", "pass"]]); - const conflicted = await triageMail(stockTriggerMail(payload), { mergeable: false }); - expect(conflicted.asked).toBe(0); - expect(conflicted.verdicts.map(({ verdict, checks }) => [verdict, checks.find((check) => check.check === "no-merge-conflicts")?.result])).toEqual([["fail", "fail"]]); - }); - - test("answers ride only on mail whose From is the hub's bare address on the deployment's domain", async () => { - const { payload } = await remailed(0.8); - const hub = createEd25519Crypto(await generateKeyPair()); - const hubSender = `github@${DOMAIN}`; - let raw: Uint8Array | undefined; - const trigger = createRunTriggerDeliverer({ - materialize: async () => ({ outcome: "materialized" as const, stepGrants: [] }), - router: { - async routeMail(_address: string, base64: string) { - raw = Buffer.from(base64, "base64"); - return true; - }, - }, - tenantDomain: async () => DOMAIN, - senderLocalPart: "github", - systemSender: { resolve: async () => ({ address: hubSender, publicKey: Buffer.from(hub.getPublicKey()).toString("hex"), sign: (input: Uint8Array) => hub.sign(input) }) }, - }); - await trigger.to(LIVE.address, JSON.stringify(payload), TENANT_ID, undefined); - const { headers, rawHeaders, parts } = decodeMail(raw!); - const fromHub = { headers, rawHeaders, parts: parts.map((part, i) => ({ contentType: part.contentType, ref: `mail-part:///run_x/m1/${i}`, text: new TextDecoder().decode(part.content) })) }; - expect((await triageMail(fromHub, { mergeable: true })).asked).toBe(0); - - // Stock binds a mail to its authenticated sender by the address `extractAddrSpec` reads, so this From passes its gate for another run. - const memberRun = formatRunAddress("run_member", DOMAIN); - const named = `"<${hubSender}>" <${memberRun}>`; - expect(extractAddrSpec(named)).toBe(memberRun); - for (const from of [named, "github@other.test"]) { - const mail = stockTriggerMail(payload); - const forged = { ...mail, headers: { ...mail.headers, from }, rawHeaders: { ...mail.rawHeaders, from: [from] } }; - expect((await triageMail(forged, { mergeable: true })).asked).toBe(1); - } - }); - - test("unsure answers are not carried", async () => { - const { first, payload } = await remailed(0.5); - expect(first.verdicts.every((verdict) => verdict.judged === undefined)).toBe(true); - expect((payload["items"] as Array<{ judged?: object }>).every((item) => item.judged === undefined)).toBe(true); - }); - - test("answers are not carried to a listener on another decision-model offering", async () => { - const { payload } = await remailed(0.8, { ...deployment("run_new", NOW), offeringId: "off_other" }); - expect((payload["items"] as Array<{ judged?: object }>).every((item) => item.judged === undefined)).toBe(true); - }); - }); - - test("heads settled under another workflow version than the live deployment's, a legacy integer included, keep their verdicts", async () => { - const { reconcile, delivered, saved } = harness({}, [head(1), head(2)]); - const settled = (number: number, workflowVersion: string | number): PrTriageRow => ({ number, headSha: `sha${number}`, status: "triaged", attempts: 0, runId: `old_${number}`, workflowVersion, firstSeenAt: LONG_AGO, updatedAt: LONG_AGO }); - const rows = [settled(1, 3), settled(2, "0.1.0-sha-00000000")]; - saved.set(REPO, rows); - await reconcile(); - expect(mailedNumbers(delivered)).toEqual([]); - expect(saved.get(REPO)).toEqual(rows); - }); - - test("a finished run's log is read once and then left out of later passes", async () => { - const logs: Logs = { [repoKey(LIVE.runId)]: { run_clean: [started(3), rendered(current(3)), completed] } }; - const { reconcile, reads, latestOf } = harness(logs, [head(3)]); - await reconcile(); - await reconcile(); - expect(reads).toHaveLength(1); - expect(latestOf[1]).toEqual([repoKey(LIVE.runId)]); - }); - test("the in-flight budget is per tenant across repositories, oldest pull requests first", async () => { const { reconcile, delivered } = harness({}, { "acme/one": heads(1, 3), "acme/two": heads(100, 3) }); await reconcile(); @@ -792,30 +488,14 @@ describe("triage reconciler", () => { expect(saved.get(REPO)?.find((row) => row.number === 1)).toMatchObject({ status: "failed", error: "delivery failed: Error: 502 Bad Gateway" }); }); - test("a batch run's verdicts settle each head it named; a batch degraded before rendering queues every head again", async () => { - const logs: Logs = { - [repoKey(LIVE.runId)]: { - run_batch: [startedBatch([1, 2, 3]), rendered({ items: [current(1), { ...current(2), headSha: "sha2-pushed" }, current(3)], summary: {} }), completed], - run_failed: [startedBatch([4, 5]), rendered({ degraded: "error", reason: "facts interrupted: abort", headSha: null }), completed], - }, - }; - const { reconcile, delivered, saved, logged } = harness(logs, heads(1, 5)); - await reconcile(); - expect(mailedNumbers(delivered)).toEqual([2, 4, 5]); - expect(logged.filter((entry) => entry["msg"] === "triage_requeued").map((entry) => entry["reason"])).toEqual(["verdict made on head sha2-pushed", "degraded: facts interrupted: abort", "degraded: facts interrupted: abort"]); - const rows = saved.get(REPO) ?? []; - expect(rows.filter((row) => row.status === "triaged").map((row) => [row.number, row.runId])).toEqual([[1, "run_batch"], [3, "run_batch"]]); - expect(rows.filter((row) => row.status === "queued").map((row) => row.number)).toEqual([2, 4, 5]); - }); - test("queued heads that have not started hold the budget until their runs complete", async () => { - const logs: Logs = { [repoKey(LIVE.runId)]: {} }; + const logs: Runs = { [REPO]: {} }; const { reconcile, delivered, saved } = harness(logs, heads(1, 12)); await reconcile(); await reconcile(); expect(mailedNumbers(delivered)).toEqual([1, 2, 3, 4, 5]); // Only a run started since the hub queued the head settles it. - for (const number of [1, 2, 3, 4, 5]) logs[repoKey(LIVE.runId)]![`run_${number}`] = [started(number, "2026-10-07T12:00:30.000Z"), rendered(current(number)), completed]; + for (const number of [1, 2, 3, 4, 5]) logs[REPO]![runKey(number, `sha${number}`)] = [ran(`run_${number}`, number, current(number), "2026-10-07T12:00:30.000Z")]; await reconcile(); expect(mailedNumbers(delivered)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); const rows = saved.get(REPO) ?? []; @@ -840,26 +520,8 @@ describe("triage reconciler", () => { expect(twice.logged).toContainEqual(expect.objectContaining({ msg: "triage_state_conflict", repo: REPO })); }); - test("a run triggered by the stock mail, decoded or stringified, is seen running and settles on its verdict", async () => { - const at = "2026-10-07T12:00:30.000Z"; - const runs: Record = {}; - const { reconcile, delivered, saved } = harness({ [repoKey(LIVE.runId)]: runs }, [head(1), head(2)]); - await reconcile(); - expect(mailedNumbers(delivered)).toEqual([1, 2]); - runs["run_mail"] = [startedBy(stockTriggerMail({ kind: "pr", repo: REPO, items: [{ prNumber: 1, headSha: "sha1" }] }), at)]; - runs["run_string"] = [startedBy(JSON.stringify(stockTriggerMail({ kind: "pr", repo: REPO, items: [{ prNumber: 2, headSha: "sha2" }] })), at)]; - await reconcile(); - expect(saved.get(REPO)!.map((row) => [row.number, row.status, row.runId])).toEqual([[1, "running", "run_mail"], [2, "running", "run_string"]]); - runs["run_mail"]!.push(rendered({ items: [current(1)] }), completed); - runs["run_string"]!.push(rendered({ items: [current(2)] }), completed); - await reconcile(); - await reconcile(); - expect(mailedNumbers(delivered)).toEqual([1, 2]); - expect(saved.get(REPO)!.map((row) => [row.number, row.status, row.runId])).toEqual([[1, "triaged", "run_mail"], [2, "triaged", "run_string"]]); - }); - test("a head still running holds one of the tenant's slots", async () => { - const logs: Logs = { [repoKey(LIVE.runId)]: { run_going: [started(1, "2026-10-07T11:55:00.000Z")] } }; + const logs = runsOf([1, ran("run_going", 1, undefined, "2026-10-07T11:55:00.000Z")]); const { reconcile, delivered } = harness(logs, heads(1, 7)); await reconcile(); expect(mailedNumbers(delivered)).toEqual([2, 3, 4, 5]); @@ -881,7 +543,7 @@ describe("triage reconciler", () => { test("a repository is listed again once a webhook named it or half an hour passed, and every pass while its webhook heard nothing that long", async () => { let clock = NOW; const activity = createRepoActivity(() => clock, { quietMs: 30 * MINUTE, graceMs: MINUTE }); - const { reconcile, listings } = harness({ [repoKey(LIVE.runId)]: settled(1, 1) }, { [REPO]: [head(1)], "acme/gadgets": [] }, { now: () => clock, activity }); + const { reconcile, listings } = harness(settled(1, 1), { [REPO]: [head(1)], "acme/gadgets": [] }, { now: () => clock, activity }); function hear(minutes: number, repo: string): void { clock = at(minutes); activity.heard(TENANT_ID, "github", repo); @@ -919,7 +581,7 @@ describe("triage reconciler", () => { let clock = NOW; const activity = createRepoActivity(() => clock, { quietMs: 30 * MINUTE, graceMs: MINUTE }); const options = { now: () => clock, activity, github: true }; - const { reconcile, listings } = harness({ [repoKey(LIVE.runId)]: settled(1, 1) }, [head(1)], options); + const { reconcile, listings } = harness(settled(1, 1), [head(1)], options); activity.heard(TENANT_ID, "github", REPO); clock = at(2); await reconcile(); @@ -936,7 +598,7 @@ describe("triage reconciler", () => { let clock = NOW; const activity = createRepoActivity(() => clock, { quietMs: 30 * MINUTE, graceMs: MINUTE }); const options: HarnessOptions = { now: () => clock, activity }; - const { reconcile, listings } = harness({ [repoKey(LIVE.runId)]: settled(1, 1) }, [head(1)], options); + const { reconcile, listings } = harness(settled(1, 1), [head(1)], options); activity.heard(TENANT_ID, "github", REPO); clock = at(25); await reconcile(); @@ -976,9 +638,9 @@ describe("triage reconciler", () => { expect(mailedNumbers(delivered)).toEqual([1]); }); - test("a tenant whose runs cannot be read asks for another pass soon", async () => { + test("a tenant whose managers' runs cannot be read asks for another pass soon", async () => { const { reconcile } = harness({}, [head(1)]); - failNextList = true; + failNextRead = true; expect(await reconcile()).toEqual({ retry: true }); }); }); diff --git a/apps/hub/src/github/triage-reconciler.ts b/apps/hub/src/github/triage-reconciler.ts index b7dcb18a6..4e74b4d51 100644 --- a/apps/hub/src/github/triage-reconciler.ts +++ b/apps/hub/src/github/triage-reconciler.ts @@ -18,8 +18,6 @@ import type { HeadChecksReader, OpenHeadsReader } from "./tenant-open-heads.js"; import type { RepoActivity } from "./repo-activity.js"; import type { LiveDeployment } from "./deployment.js"; import type { ManagerRuns } from "./manager-runs.js"; -import { mergeObservedRuns, type ObservedRuns } from "./triage-runs.js"; -import type { TenantObserver } from "./tenant-observer.js"; import type { VerdictRecord, VerdictRecordStore } from "./verdict-record-store.js"; import { TriageStateConflictError, type LoadedTriageState, type TriageStateStore, type TriageStateVersion } from "./triage-state-store.js"; @@ -41,10 +39,8 @@ export type TriageReconcilerDeps = { * finishes and for a completed check run whose event was lost or not delivered. */ headChecks: HeadChecksReader; - /** Recorded verdicts, so a head is polled whichever deployment's run, live or released since, gave its verdict. */ + /** Recorded verdicts, whose CI a settled head is polled for. */ verdicts: Pick; - /** Reads and records the live deployments' runs, shared with the recording loop. */ - observeTenant: TenantObserver["observe"]; /** The host's pull request managers, read from their claims and the verdicts they reported. */ managerRuns: ManagerRuns; store: TriageStateStore; @@ -56,7 +52,7 @@ export type TriageReconcilerDeps = { }; type EnabledRepo = { record: RepoRecord; policy: RepoPolicy }; -type TenantPass = { tenantId: string; deployment: LiveDeployment; openHeads: OpenHeadsReader; runs: ObservedRuns }; +type TenantPass = { tenantId: string; openHeads: OpenHeadsReader }; type LoadedRepo = EnabledRepo & (StoredPack | PolicyUnreadable) & { prs: OpenPr[]; stored: LoadedTriageState; ci: Map }; /** @@ -145,9 +141,9 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { return prs; } - /** The newest verdict on the head, from the live logs or as recorded, since the run that gave it may be on a released deployment. */ - function latestVerdict(pass: TenantPass, recorded: readonly VerdictRecord[], repo: string, row: PrTriageRow) { - return [...pass.runs.verdicts, ...recorded].filter((verdict) => verdict.repo === repo && verdict.number === row.number && verdict.headSha === row.headSha).sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime())[0]; + /** The newest verdict recorded on the head. */ + function latestVerdict(recorded: readonly VerdictRecord[], repo: string, row: PrTriageRow) { + return recorded.filter((verdict) => verdict.repo === repo && verdict.number === row.number && verdict.headSha === row.headSha).sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime())[0]; } /** Queues a settled head whose CI the verdict waited on once its build statuses are no longer pending. */ @@ -158,7 +154,7 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { const recorded = (await deps.verdicts.latest(pass.tenantId, triaged.map((row) => ({ repo: name, number: row.number })))).filter((verdict) => verdict.source === deps.source); for (const row of triaged) { const key = `${pass.tenantId}\n${name}\n${runKey(row.number, row.headSha)}`; - const verdict = latestVerdict(pass, recorded, name, row); + const verdict = latestVerdict(recorded, name, row); if (!verdict || !waitsOnCi(verdict)) { buildPolls.delete(key); continue; @@ -280,7 +276,7 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { for (const queued of claimed.enqueue) { const headSha = headShaOf(queued); const extra = queued.events === undefined - ? { items: [{ prNumber: queued.number, headSha, ...(queued.judged && { judged: queued.judged }) }] } + ? { items: [{ prNumber: queued.number, headSha }] } : { prNumber: queued.number, headSha, events: queued.events }; const mailed = { ...extra, ...(queued.noWait && { noWait: true }) }; const payload = "pack" in repo ? mailPayload(repo.record, repo.policy, repo, mailed) : unreadablePayload(repo.record, repo.policy, repo, mailed); @@ -305,8 +301,6 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { deps.log({ level: "warn", msg: "triage_reconcile_skipped", tenantId: tenant.id, reason: "no_listener" }); return false; } - // Only live deployments' and recent workers' logs are read; heads triaged under a released one stay settled in the stored state. - const { observed, workers } = await deps.observeTenant(tenant.id, tenant.domain); let openHeads: OpenHeadsReader | undefined; try { openHeads = await deps.openHeadsFor(tenant.id); @@ -319,8 +313,7 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { deps.log({ level: "warn", msg: "triage_reconcile_skipped", tenantId: tenant.id, reason: "no_source_credential" }); return false; } - const runs = mergeObservedRuns([...observed, ...workers]); - const pass: TenantPass = { tenantId: tenant.id, deployment: listener, openHeads, runs }; + const pass: TenantPass = { tenantId: tenant.id, openHeads }; const loaded: LoadedRepo[] = []; let failed = false; for (const repo of repos) { @@ -332,14 +325,12 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { } } const pulls = loaded.flatMap((repo) => repo.prs.map((pr) => ({ source: deps.source, repo: repo.record.name, number: pr.number }))); - const managed = await deps.managerRuns(tenant.id, pulls, pass.deployment.workflowVersion); + const managed = await deps.managerRuns(tenant.id, pulls); const plans = planTenant({ repos: loaded.map((repo) => ({ name: repo.record.name, prs: repo.prs, rows: repo.stored.rows, ci: repo.ci })), - runs: mergeObservedRuns([pass.runs, managed]).byRepo, + runs: managed.byRepo, now: deps.now(), policy: deps.policy, - workflowVersion: pass.deployment.workflowVersion, - offeringId: pass.deployment.offeringId, }); for (const repo of loaded) { try { diff --git a/apps/hub/src/github/triage-runs.ts b/apps/hub/src/github/triage-runs.ts index ba7fab1e9..f0466a91f 100644 --- a/apps/hub/src/github/triage-runs.ts +++ b/apps/hub/src/github/triage-runs.ts @@ -1,61 +1,16 @@ -// The live pr-triage deployment's runs, read from its committed run event log -// and keyed by the pull request head their trigger named. -import { and, eq, inArray } from "drizzle-orm"; +// What the audit log and held Dos read from a pr-triage run's event log: the +// approval it carried out and the verdict it made for a pull request. import { type } from "arktype"; -import { APPROVE_DO_KIND, isFromTriageSender, repoSourceOf, triggerRequestOf, type RepoSource } from "@corbits/triage-contracts"; +import { APPROVE_DO_KIND, isFromTriageSender, triggerRequestOf } from "@corbits/triage-contracts"; import { asRecord } from "@corbits/common"; -import { schema, type DB } from "@intx/db"; -import { formatRunAddress } from "@intx/types"; -import { classifyTerminalEvent, WORKFLOW_RUN_REF, workflowRunRepoIdForAddress, type WorkflowRunEvent, type WorkflowRunReader } from "@intx/hub-sessions"; -import { heldForCi, unsettledBy } from "@corbits/triage-workflows/logic/settled"; -import type { Judged } from "@corbits/triage-workflows/logic/item"; -import type { LiveDeployment } from "./deployment.js"; -import { runKey, type ObservedRun } from "./reconcile-plan.js"; -import type { ApprovedDo, RecordedVerdict } from "./verdict-record-store.js"; -import { DoResult, doResultsOf, eventOf, mirrorOutcomesOf, ruledOf, summaryOf, withResults, type Ruled, type Summary, type Trigger } from "./verdict-row.js"; +import { classifyTerminalEvent, type WorkflowRunEvent } from "@intx/hub-sessions"; +import type { ApprovedDo } from "./verdict-record-store.js"; +import { DoResult, doResultsOf, withResults } from "./verdict-row.js"; const INLINE_PREFIX = "inline:"; -const NO_VERDICT = "run completed without a verdict"; -type Unsettled = Pick; -type Terminal = Unsettled & { status: Exclude }; -type Head = { trigger: Trigger; terminal?: Terminal; summary?: Summary; judged?: Judged }; -/** No heads when the trigger names no pull request head, such as a portal run typed without a sha; a catch-up mail names several. */ /** An approval run names no head; `approved` is set once it ended, empty when it wrote nothing. `principalId` is who approved. */ export type Approval = { repo: string; number: number; principalId?: string; approved?: ApprovedDo[] }; -type KnownRun = { - anchorRunId: string; - heads: Head[]; - /** A backlog mail names no heads, so its run is read until it ends, when its verdicts name them. */ - backlog?: { ended: boolean }; - approval?: Approval; - /** The newest event read; a running run is read again once its log moves past it. */ - seq: number; -}; - -export type ObservedRuns = { - byRepo: Map>; - /** Runs in the log, one per mail it received, including those that name no pull request head. */ - runCount: number; - /** One per head a run made a verdict for, from when its verdict step completed, however the run then ends. */ - verdicts: RecordedVerdict[]; - approved: ApprovedDo[]; -}; - -export function isRunning(runs: ObservedRuns): boolean { - return [...runs.byRepo.values()].some((byHead) => [...byHead.values()].some((list) => list.some((run) => run.status === "running"))); -} - -/** Statuses the stock lifecycle settled for these runs, which it can do without a terminal event in the log. */ -export type ReadSettled = (anchorRunId: string, runIds: string[]) => Promise>; - -export type TriageRunsDeps = { - runReader: WorkflowRunReader; - readSettled: ReadSettled; - /** Runs remembered at most; the oldest is forgotten first and read again while it is still in the log. */ - maxKnownRuns: number; -}; - function parseJson(value: unknown): unknown { if (typeof value !== "string") return value; try { @@ -65,28 +20,6 @@ function parseJson(value: unknown): unknown { } } -function headOf(item: unknown, repo: string, source: RepoSource, startedAt: string, event: string | undefined): Trigger[] { - const number = asRecord(item)?.["prNumber"]; - const headSha = asRecord(item)?.["headSha"]; - if (typeof number !== "number" || typeof headSha !== "string" || headSha === "") return []; - return [{ repo, source, number, headSha, startedAt, ...(event !== undefined && { event }) }]; -} - -function isBacklog(started: Record): boolean { - return triggerRequestOf(asRecord(started["trigger"])?.["payload"])?.["kind"] === "backlog"; -} - -/** The heads a mail named: one as `prNumber`/`headSha`, or several as `items` of the same. */ -function triggersOf(started: Record): Trigger[] { - const payload = triggerRequestOf(asRecord(started["trigger"])?.["payload"]); - const repo = payload?.["repo"]; - const startedAt = started["at"]; - if (payload?.["kind"] !== "pr" || typeof repo !== "string" || typeof startedAt !== "string") return []; - const items = payload["items"]; - const source = repoSourceOf(payload["repoSource"]); - return (Array.isArray(items) ? items : [payload]).flatMap((item) => headOf(item, repo, source, startedAt, eventOf(payload))); -} - /** The pull request an approval mail from the hub's sender names a Do of; pr-triage takes any other mail as no approval. */ function approvalOf(started: Record): Approval | undefined { const mail = asRecord(started["trigger"])?.["payload"]; @@ -99,7 +32,7 @@ function approvalOf(started: Record): Approval | undefined { return { repo, number, ...(typeof principalId === "string" && { principalId }) }; } -function terminalStatus(events: readonly WorkflowRunEvent[]): Terminal["status"] | undefined { +function terminalStatus(events: readonly WorkflowRunEvent[]): string | undefined { const last = events.at(-1); const terminal = last && classifyTerminalEvent(last.type); return terminal?.terminal ? terminal.status : undefined; @@ -121,8 +54,6 @@ export function approvalIn(events: readonly WorkflowRunEvent[]): Approval | unde const VERDICT_STEPS = new Set(["evaluate", "evaluateRules", "render", "renderJudged", "renderDet", "renderAll"]); /** The PR workflow's `decide` replaces the verdict's Dos with those it chose. */ const DECIDE_STEPS = new Set(["decide"]); -const RULES_STEPS = new Set(["rules"]); -const MIRROR_STEPS = new Set(["mirror", "mirrorRules"]); const EXECUTE_STEPS = new Set(["execute", "executeRules"]); const APPROVED_STEPS = new Set(["executeApproved"]); @@ -178,203 +109,6 @@ function verdictsOf(events: readonly WorkflowRunEvent[]): Array withResults(item, results)); } -function ruledIn(events: readonly WorkflowRunEvent[]): Map { - return ruledOf(inlineOutput(lastCompleted(events, RULES_STEPS))); -} - -function mirrorOutcomes(events: readonly WorkflowRunEvent[]): Map> { - return mirrorOutcomesOf(inlineOutput(lastCompleted(events, MIRROR_STEPS))); -} - export function verdictFor(events: readonly WorkflowRunEvent[], repo: string, number: number): Record | undefined { return verdictsOf(events).find((verdict) => verdict?.["repo"] === repo && verdict["number"] === number); } - -/** The judge answers a verdict made on the trigger's head says a later run of it can reuse. */ -function judgedOf(verdict: Record, trigger: Trigger): Judged | undefined { - const judged = asRecord(verdict["judged"]); - if (!judged || verdict["headSha"] !== trigger.headSha) return undefined; - const answers = Object.entries(judged).filter((entry): entry is [string, string] => typeof entry[1] === "string"); - return answers.length > 0 ? Object.fromEntries(answers) : undefined; -} - -function terminalOf(status: Terminal["status"], verdict: Record | undefined, headSha: string): Terminal { - if (!verdict) return { status, unsettled: NO_VERDICT }; - return { status, ...unsettledBy(verdict, headSha), ...(heldForCi(verdict) && { heldForCi: true }) }; -} - -function headVerdictOf(verdicts: readonly T[], heads: number, index: number): T | undefined { - if (verdicts.length === heads) return verdicts[index]; - if (verdicts.length === 1) return verdicts[0]; - return undefined; -} - -type Paired = { trigger: Trigger; verdict: Record | undefined }; - -/** The verdict step's output is the verdict: one per head in mail order, or one degraded verdict for every head. Any other count pairs nothing, so no head settles on another's verdict. */ -function mailPairs(started: Record, verdicts: ReadonlyArray | undefined>): Paired[] { - const triggers = triggersOf(started); - return triggers.map((trigger, i) => ({ trigger, verdict: headVerdictOf(verdicts, triggers.length, i) })); -} - -/** A backlog mail names no heads: they are the pull requests its run made a verdict for, each paired with its own. */ -function backlogPairs(started: Record, verdicts: ReadonlyArray | undefined>): Paired[] { - const payload = triggerRequestOf(asRecord(started["trigger"])?.["payload"]); - const repo = payload?.["repo"]; - const startedAt = started["at"]; - if (typeof repo !== "string" || typeof startedAt !== "string") return []; - const source = repoSourceOf(payload?.["repoSource"]); - return verdicts.flatMap((verdict) => headOf({ prNumber: verdict?.["number"], headSha: verdict?.["headSha"] }, repo, source, startedAt, undefined).map((trigger) => ({ trigger, verdict }))); -} - -function headsOf(events: readonly WorkflowRunEvent[], started: Record): Head[] { - const status = terminalStatus(events); - const verdicts = verdictsOf(events); - const verdictAt = lastCompleted(events, VERDICT_STEPS)?.body["at"]; - const mirrored = mirrorOutcomes(events); - const ruled = ruledIn(events); - const pairs = isBacklog(started) ? backlogPairs(started, verdicts) : mailPairs(started, verdicts); - // The verdict is kept from when its step completed, so a run that fails after it still shows it. - return pairs.map(function headVerdict({ trigger, verdict }) { - const summary = verdict && summaryOf(verdict, verdictAt, trigger, mirrored, ruled.get(`${trigger.repo}#${trigger.number}`)); - const kept = { trigger, ...(summary && { summary }) }; - if (status === undefined) return kept; - if (status !== "completed") return { ...kept, terminal: { status } }; - const judged = verdict && judgedOf(verdict, trigger); - return { ...kept, terminal: terminalOf(status, verdict, trigger.headSha), ...(judged && { judged }) }; - }); -} - -export type ObserveRuns = (anchorRunId: string, domain: string) => Promise; - -type Stamp = Pick; - -function stamped(runs: ObservedRuns, stamp: Stamp): ObservedRuns { - const byRepo: ObservedRuns["byRepo"] = new Map(); - for (const [repo, byHead] of runs.byRepo) { - byRepo.set(repo, new Map([...byHead].map(([key, list]) => [key, list.map((run) => ({ ...run, ...stamp }))]))); - } - return { ...runs, byRepo }; -} - -/** Each deployment's runs, in the order given, stamped with the workflow version the deployment runs and the decision-model offering it resolved. */ -export async function observeDeployments(observe: ObserveRuns, deployments: readonly Pick[], domain: string): Promise { - const observed: ObservedRuns[] = []; - for (const { runId, workflowVersion, offeringId } of deployments) { - const runs = await observe(runId, domain); - const stamp = { ...(workflowVersion !== undefined && { workflowVersion }), ...(offeringId !== undefined && { offeringId }) }; - observed.push(Object.keys(stamp).length === 0 ? runs : stamped(runs, stamp)); - } - return observed; -} - -export function mergeObservedRuns(observed: readonly ObservedRuns[]): ObservedRuns { - const byRepo: ObservedRuns["byRepo"] = new Map(); - for (const runs of observed) { - for (const [repo, byHead] of runs.byRepo) { - const merged = byRepo.get(repo) ?? new Map(); - byRepo.set(repo, merged); - for (const [key, list] of byHead) merged.set(key, [...(merged.get(key) ?? []), ...list]); - } - } - const runCount = observed.reduce((sum, runs) => sum + runs.runCount, 0); - return { byRepo, runCount, verdicts: observed.flatMap((runs) => runs.verdicts), approved: observed.flatMap((runs) => runs.approved) }; -} - -export function createSettledStatusReader(db: DB["db"]): ReadSettled { - return async function readSettled(anchorRunId, runIds) { - if (runIds.length === 0) return new Map(); - const { workflowRun } = schema; - const rows = await db - .select({ id: workflowRun.id, status: workflowRun.status }) - .from(workflowRun) - .where(and(eq(workflowRun.anchorRunId, anchorRunId), inArray(workflowRun.id, runIds), inArray(workflowRun.status, ["failed", "cancelled"]))); - return new Map(rows.map((row) => [row.id, row.status as Terminal["status"]])); - }; -} - -export function createTriageRuns(deps: TriageRunsDeps) { - // Insertion-ordered, so the first key is the oldest. - const known = new Map(); - - function remember(runId: string, run: KnownRun): void { - known.delete(runId); - known.set(runId, run); - if (known.size > deps.maxKnownRuns) known.delete(known.keys().next().value!); - } - - function running(run: KnownRun): boolean { - return run.heads.some((head) => head.terminal === undefined) || (run.approval !== undefined && run.approval.approved === undefined) || run.backlog?.ended === false; - } - - /** A finished log never changes, so a settled run is never read again. */ - function settledRuns(anchorRunId: string): Set { - const ids = new Set(); - for (const [runId, run] of known) { - if (run.anchorRunId === anchorRunId && !running(run)) ids.add(runId); - } - return ids; - } - - return async function observeRuns(anchorRunId: string, domain: string): Promise { - const repoId = workflowRunRepoIdForAddress(formatRunAddress(anchorRunId, domain)); - const settledIds = settledRuns(anchorRunId); - const { events: latest } = await deps.runReader.readLatestRunEvents( - repoId, - WORKFLOW_RUN_REF, - (runId) => runId !== anchorRunId && !settledIds.has(runId), - ); - const open: string[] = []; - for (const [runId, event] of latest) { - const ended = event !== null && classifyTerminalEvent(event.type).terminal; - if (!ended && known.get(runId)?.seq === (event?.seq ?? -1)) { - open.push(runId); - continue; - } - const events = await deps.runReader.readRunEvents(repoId, WORKFLOW_RUN_REF, runId); - const started = events.find((e) => e.type === "RunStarted"); - // A run that has not committed RunStarted yet is read again next pass. - if (!started) continue; - const heads = headsOf(events, started.body); - const approval = approvalOf(started.body); - const backlog = isBacklog(started.body) && { backlog: { ended: terminalStatus(events) !== undefined } }; - const run: KnownRun = { anchorRunId, heads, ...(approval && { approval: approvedOf(approval, events) }), ...backlog, seq: events.at(-1)?.seq ?? -1 }; - remember(runId, run); - if (running(run)) open.push(runId); - } - const settled = await deps.readSettled(anchorRunId, open); - for (const [runId, status] of settled) { - const run = known.get(runId); - if (!run) continue; - const approval = run.approval && { ...run.approval, approved: run.approval.approved ?? [] }; - remember(runId, { ...run, heads: run.heads.map((head) => ({ ...head, terminal: { status } })), ...(approval && { approval }), ...(run.backlog && { backlog: { ended: true } }) }); - } - - const byRepo: ObservedRuns["byRepo"] = new Map(); - const verdicts: RecordedVerdict[] = []; - const approved: ApprovedDo[] = []; - for (const runId of [...settledIds, ...latest.keys()]) { - approved.push(...(known.get(runId)?.approval?.approved ?? [])); - for (const { trigger, terminal, summary, judged } of known.get(runId)?.heads ?? []) { - const { repo, number, headSha, startedAt } = trigger; - if (summary) verdicts.push({ runId, repo, number, headSha, startedAt: new Date(startedAt), ...summary, runStatus: terminal?.status ?? "running" }); - const byHead = byRepo.get(repo) ?? new Map(); - byRepo.set(repo, byHead); - const key = runKey(number, headSha); - const seen: ObservedRun = { - runId, - source: trigger.source, - status: terminal?.status ?? "running", - startedAt, - ...(terminal?.unsettled !== undefined && { unsettled: terminal.unsettled }), - ...(terminal?.unconfirmed && { unconfirmed: true }), - ...(terminal?.heldForCi && { heldForCi: true }), - ...(trigger.event !== undefined && { trigger: trigger.event }), - ...(judged && { judged }), - }; - byHead.set(key, [...(byHead.get(key) ?? []), seen]); - } - } - return { byRepo, runCount: settledIds.size + latest.size, verdicts, approved }; - }; -} diff --git a/apps/hub/src/github/triage-verdicts.postgres.test.ts b/apps/hub/src/github/triage-verdicts.postgres.test.ts index f17d98d9e..c5db36a15 100644 --- a/apps/hub/src/github/triage-verdicts.postgres.test.ts +++ b/apps/hub/src/github/triage-verdicts.postgres.test.ts @@ -12,7 +12,7 @@ import { TOKEN_CREDENTIAL } from "../bitbucket-datacenter/records.js"; import { createGithubOpenPulls, createOpenPullsCache, GITHUB_OPEN_PULLS_PATH, type RepoPulls } from "./open-pulls.js"; import type { RepoRecord } from "./tenant-config.js"; import type { ObservedRun } from "./reconcile-plan.js"; -import { createTenantVersions, type TenantObservation } from "./tenant-observer.js"; +import type { ObservedRuns } from "./manager-runs.js"; import { createTriageVerdicts } from "./triage-verdicts.js"; import { triageWorkflowFor } from "./source-workflow.js"; import { createVerdictRecordStore, type RecordedVerdict } from "./verdict-record-store.js"; @@ -43,26 +43,24 @@ function verdict(runId: string, number: number, at: number, fields: Partial): ObservedRuns { + return { byRepo: new Map([[REPO, new Map(byHead)]]) }; +} + +const NO_RUNS: ObservedRuns = { byRepo: new Map() }; + +/** The managers' runs: #8's newest run failed after its verdict, #9 and #10 are still running and #6's run failed early; #7 has no run, but waits for CI. */ +const MANAGED = observed([ + ["8@sha_run_3", [live("run_3", "failed", minutes(9))]], + ["9@sha9", [live("run_9", "running", minutes(-5), "pull_request:synchronize")]], + ["10@sha10", [live("run_10", "running", minutes(-120))]], + ["6@sha6", [live("run_6", "failed", minutes(-3))]], +]); const OPEN: RepoPulls[] = [ { repo: REPO, source: "github", prs: [6, 7, 8, 9, 10].map((number) => ({ number, title: `PR ${number}`, author: "ada", draft: false, sha: `sha${number}`, branch: null, updatedAt: minutes(0).toISOString(), labels: [], url: null })) }, @@ -122,9 +120,8 @@ test("PostgreSQL: the latest verdict of each open pull request, with the run in }; let session: { user: { id: string } } | null = { user: { id: "usr_member" } }; const grants: GrantRule[] = [{ id: "grt_runs", resource: "workflow-run:*", action: "read", effect: "allow", origin: "system", conditions: null, expiresAt: null, roleId: null, principalId: "prn_verdicts" }]; - let ready = false; let now = minutes(0); - const versions = createTenantVersions(); + let managed = MANAGED; // The portal's github-open-pulls listing; the workspace has no GitHub credential, so the route never lists on its own. const openPulls = createOpenPullsCache(() => minutes(0)); await openPulls.read(TENANT, async () => OPEN); @@ -134,9 +131,7 @@ test("PostgreSQL: the latest verdict of each open pull request, with the run in onKeyMoved: noKeyPush, authorize: async () => true, verdicts, - observer: { last: () => OBSERVATION }, - ready: () => ready, - versions, + managerRuns: async () => managed, openPulls, list: () => async () => { throw new Error("the workspace has no host credential"); }, triageState: { load: async (_tenant, key) => ({ rows: key === REPO ? waitingRows : [], version: null }) }, @@ -147,10 +142,6 @@ test("PostgreSQL: the latest verdict of each open pull request, with the run in return app.request(`/api/integrations/triage-verdicts/${TENANT}${query}`, { headers }); } - // Until the hub read the live logs since it started, any pull request may have a run going. - const warming = TriageVerdicts.assert(await (await get()).json()); - expect(new Set(warming.pulls.map((pull) => pull.run?.status))).toEqual(new Set(["unknown", "waiting-ci"])); - ready = true; // The listing is past 30 seconds old; without the GitHub credential the route keeps using it. now = minutes(1); const res = await get(); @@ -185,10 +176,10 @@ test("PostgreSQL: the latest verdict of each open pull request, with the run in const etag = res.headers.get("etag")!; expect((await get("", { "sec-fetch-site": "same-origin", "if-none-match": etag })).status).toBe(304); expect((await get()).status).toBe(200); + expect(reads).toBe(1); + managed = observed([...MANAGED.byRepo.get(REPO)!].filter(([key]) => key !== "6@sha6")); + expect((await get("", { "sec-fetch-site": "same-origin", "if-none-match": etag })).status).toBe(200); expect(reads).toBe(2); - versions.bump(TENANT); - expect((await get("", { "sec-fetch-site": "same-origin", "if-none-match": etag })).status).toBe(304); - expect(reads).toBe(3); const closed = TriageVerdicts.assert(await (await get(`?repo=${REPO}&number=5`)).json()); expect(closed.pulls.map((pull) => [pull.number, pull.verdict?.runId])).toEqual([[5, "run_5"]]); @@ -228,9 +219,7 @@ test("PostgreSQL: pull requests of a repository with triage off are listed only onKeyMoved: noKeyPush, authorize: async () => true, verdicts, - observer: { last: () => undefined }, - ready: () => true, - versions: createTenantVersions(), + managerRuns: async () => NO_RUNS, openPulls, list: () => async () => { throw new Error("the workspace has no host credential"); }, triageState: { load: async () => ({ rows: [], version: null }) }, @@ -281,9 +270,7 @@ test("PostgreSQL: a host with no credential costs only its own repositories, whi const triageVerdicts = createTriageVerdicts({ ...deps, verdicts, - observer: { last: () => undefined }, - ready: () => true, - versions: createTenantVersions(), + managerRuns: async () => NO_RUNS, triageState: { load: async () => ({ rows: [], version: null }) }, now: () => now, }); @@ -361,7 +348,7 @@ test("PostgreSQL: the same repository and number on both hosts come back separat verdict("run_github_off", 2, old, { repo: off }), verdict("run_bitbucket_off", 2, old, { repo: off }), ]); - let observation: TenantObservation | undefined; + let managed = NO_RUNS; let listedAt = 0; const openPulls = createOpenPullsCache(() => new Date(NOW + listedAt)); const triageVerdicts = createTriageVerdicts({ @@ -370,9 +357,7 @@ test("PostgreSQL: the same repository and number on both hosts come back separat onKeyMoved: noKeyPush, authorize: async () => true, verdicts: store, - observer: { last: () => observation }, - ready: () => true, - versions: createTenantVersions(), + managerRuns: async () => managed, openPulls, list: () => async () => { throw new Error("the workspace has no host credential"); }, triageState: { load: async () => ({ rows: [], version: null }) }, @@ -409,11 +394,7 @@ test("PostgreSQL: the same repository and number on both hosts come back separat await listing(1_000, "unreachable"); expect(rows(await get())).toEqual([["github", REPO, 8, "run_github", minutes(-1).toISOString(), null]]); - observation = { - deployments: [], - observed: [{ byRepo: new Map([[REPO, new Map([["8@sha_live", [{ ...live("run_bitbucket_live", "running", minutes(-1)), source: "bitbucket-datacenter" as const }]]])]]), runCount: 1, verdicts: [], approved: [] }], - workers: [], - }; + managed = observed([["8@sha_live", [{ ...live("run_bitbucket_live", "running", minutes(-1)), source: "bitbucket-datacenter" as const }]]]); await listing(2_000); const running = await get(); expect(rows(running).filter((row) => row[1] === REPO).map((row) => [row[0], row[5]])).toEqual([["bitbucket-datacenter", "running"], ["github", null]]); @@ -421,6 +402,11 @@ test("PostgreSQL: the same repository and number on both hosts come back separat expect((await get(`?repo=${REPO}&number=8&source=bitbucket-datacenter`)).pulls.map((pull) => pull.verdict?.runId)).toEqual(["run_bitbucket"]); expect((await get(`?repo=${REPO}&number=8`)).pulls.map((pull) => pull.verdict?.runId)).toEqual(["run_github"]); + // A manager up between passes, such as one waiting on a reviewer, runs nothing, so its pull request shows no run. + const { passAt: _passAt, ...between } = live("run_bitbucket_live", "running", minutes(-1)); + managed = observed([["8@sha_live", [{ ...between, source: "bitbucket-datacenter" as const }]]]); + expect((await get(`?repo=${REPO}&number=8&source=bitbucket-datacenter`)).pulls.map((pull) => pull.run)).toEqual([null]); + // A name only Bitbucket holds is found without naming its host. await db.insert(schema.workflowRun).values({ id: "run_bitbucket_only", definitionId: "wfd_bitbucket", anchorRunId: "run_bitbucket_only", tenantId: TENANT, status: "completed" }); await store.record(TENANT, [verdict("run_bitbucket_only", 3, 0, { repo: only })]); diff --git a/apps/hub/src/github/triage-verdicts.ts b/apps/hub/src/github/triage-verdicts.ts index 751b74090..c16b8e7af 100644 --- a/apps/hub/src/github/triage-verdicts.ts +++ b/apps/hub/src/github/triage-verdicts.ts @@ -1,10 +1,9 @@ -// The latest verdict of each open pull request, with the pr-triage run in -// flight for it, so the portal lists pull requests without reading run logs. -// Reads only: the tables, the tenant observer's last read of the live logs, the -// triage state of heads waiting for CI and the open pull requests GitHub last -// listed. The body is kept per workspace until any of those changes, so polling -// tabs are answered from memory. It needs the grant the portal needed to read -// run logs, `workflow-run:*` read. +// The latest verdict of each open pull request, with its manager's run in flight, +// so the portal lists pull requests without reading run logs. Reads only: the +// tables, the managers' claims, the triage state of heads waiting for CI and the +// open pull requests GitHub last listed. The body is kept per workspace until any +// of those changes, so polling tabs are answered from memory. It needs the grant +// the portal needed to read run logs, `workflow-run:*` read. import { createHash } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { type } from "arktype"; @@ -18,7 +17,7 @@ import type { Member } from "../integrations/app.js"; import { runKeyNumber, type ObservedRun } from "./reconcile-plan.js"; import { repoRecords, sourceOf, triageNs, triageStateKey, type RepoRecord } from "./tenant-config.js"; import type { TriageStateStore } from "./triage-state-store.js"; -import type { TenantObservation, TenantObserver, TenantVersions } from "./tenant-observer.js"; +import type { ManagerRuns, ObservedRuns } from "./manager-runs.js"; import { type HostedPull, type HostedRepo, type HostedVerdict, type VerdictRecord, type VerdictRecordStore } from "./verdict-record-store.js"; export const TRIAGE_VERDICTS_PATH = "/api/integrations/triage-verdicts"; @@ -57,9 +56,7 @@ type Run = NonNullable; export type TriageVerdictsDeps = PortalCredentialDeps & { verdicts: Pick; - observer: Pick; - ready: (tenantId: string) => boolean; - versions: Pick; + managerRuns: ManagerRuns; openPulls: OpenPullsCache; list: OpenPrsLister; triageState: Pick; @@ -79,17 +76,15 @@ function pullKey(pull: HostedPull): string { return `${hostKey(pull)}#${pull.number}`; } -function livePulls(observation: TenantObservation | undefined): Map { +function livePulls(runs: ObservedRuns): Map { const byPull = new Map(); - for (const runs of observation ? [...observation.observed, ...observation.workers] : []) { - for (const [repo, byHead] of runs.byRepo) { - for (const [key, list] of byHead) { - for (const run of list) { - const ref = { repo, source: run.source, number: runKeyNumber(key) }; - const pull = byPull.get(pullKey(ref)) ?? { ...ref, runs: [] }; - pull.runs.push(run); - byPull.set(pullKey(ref), pull); - } + for (const [repo, byHead] of runs.byRepo) { + for (const [key, list] of byHead) { + for (const run of list) { + const ref = { repo, source: run.source, number: runKeyNumber(key) }; + const pull = byPull.get(pullKey(ref)) ?? { ...ref, runs: [] }; + pull.runs.push(run); + byPull.set(pullKey(ref), pull); } } } @@ -97,18 +92,20 @@ function livePulls(observation: TenantObservation | undefined): Map PR_TRIAGE_STUCK_RUN_MS; return { runId, startedAt, status: stuck ? "stuck" : "running" }; } /** Waiting for CI holds the head's run back, so an earlier head's run says nothing about it. */ -function runFor(waiting: WaitingPull | undefined, live: ObservedRun | undefined, now: Date, ready: boolean): Run | null { - return waiting === undefined ? runOf(live, now, ready) : { status: "waiting-ci", since: waiting.since, headSha: waiting.headSha }; +function runFor(waiting: WaitingPull | undefined, live: ObservedRun | undefined, now: Date): Run | null { + return waiting === undefined ? runOf(live, now) : { status: "waiting-ci", since: waiting.since, headSha: waiting.headSha }; } const STATE_VERDICTS: Record = { "ready-monitoring": "pass", blocked: "fail", "needs-decision": "fail", "needs-author-update": "fail" }; @@ -198,7 +195,7 @@ export function createTriageVerdicts(deps: TriageVerdictsDeps) { return at; } - /** The connected repositories, each on its host, with a verdict since `since`, or ever, or a run in a live log. */ + /** The connected repositories, each on its host, with a verdict since `since`, or ever, or a manager's run. */ async function reposOf(tenantId: string, connected: readonly RepoRecord[], live: Map, since?: Date): Promise { const seen = new Set((await deps.verdicts.reposSince(tenantId, connected.map((record) => record.name), since)).map(hostKey)); for (const pull of live.values()) seen.add(hostKey(pull)); @@ -233,13 +230,12 @@ export function createTriageVerdicts(deps: TriageVerdictsDeps) { reposOf(tenantId, connected, live, new Date(now.getTime() - RECENT_MS)), deps.verdicts.anyScored(tenantId), ]); - const ready = deps.ready(tenantId); return { pulls: pulls.map((ref) => ({ ...ref, activityAt: activity.get(pullKey(ref)) ?? null, verdict: verdictOf(byPull.get(pullKey(ref))), - run: runFor(waiting.get(pullKey(ref)), live.get(pullKey(ref))?.runs[0], now, ready), + run: runFor(waiting.get(pullKey(ref)), live.get(pullKey(ref))?.runs[0], now), })), triaged, active, @@ -259,8 +255,7 @@ export function createTriageVerdicts(deps: TriageVerdictsDeps) { } /** Lists the pull requests of `enabled` repositories and the `held` ones; `connected`, every connected one, still decides which count as triaged. */ - async function openList(tenantId: string, enabled: readonly RepoRecord[], held: readonly HostedPull[], connected: readonly RepoRecord[], listed: OpenPullsRead | undefined, live: Map): Promise { - const open = openNumbers(listed, enabled, held); + async function openList(tenantId: string, enabled: readonly RepoRecord[], connected: readonly RepoRecord[], open: Map }>, live: Map): Promise { const unknown = enabled.filter((record) => !open.has(triageStateKey(record))); const unlisted = new Set(unknown.map(triageStateKey)); const pairs = [...open.values()].flatMap(({ repo, numbers }) => [...numbers].map((number) => ({ repo, number }))); @@ -281,23 +276,26 @@ export function createTriageVerdicts(deps: TriageVerdictsDeps) { const tenant = await deps.db.query.tenant.findFirst({ where: eq(schema.tenant.id, tenantId) }); if (!tenant) return failure(409, "workspace_unconfigured", "The workspace is not set up yet."); const connected = repoRecords(triageNs(tenant.config)).filter((record) => record.connected); - const live = livePulls(deps.observer.last(tenantId)); if (query.repo !== undefined && query.number !== undefined) { const record = connectedRecord(connected, query.repo, query.source); if (!record) return failure(404, "not_found", "That repository is not connected."); const pull = { repo: query.repo, number: query.number, source: sourceOf(record) }; - const [latest, waiting] = await Promise.all([deps.verdicts.latestFor(tenantId, pull), waitingOf(tenantId, [record])]); - const one = await build(tenantId, latest ? [latest] : [], new Map([...live].filter(([key]) => key === pullKey(pull))), new Map([...waiting].filter(([key]) => key === pullKey(pull))), () => true, connected); - return respond(req, bodyOf(one.pulls.length > 0 ? one : { ...one, pulls: [{ ...pull, activityAt: null, verdict: null, run: runOf(undefined, deps.now(), deps.ready(tenantId)) }] })); + const [latest, waiting, runs] = await Promise.all([deps.verdicts.latestFor(tenantId, pull), waitingOf(tenantId, [record]), deps.managerRuns(tenantId, [pull])]); + const one = await build(tenantId, latest ? [latest] : [], new Map([...livePulls(runs)].filter(([key]) => key === pullKey(pull))), new Map([...waiting].filter(([key]) => key === pullKey(pull))), () => true, connected); + return respond(req, bodyOf(one.pulls.length > 0 ? one : { ...one, pulls: [{ ...pull, activityAt: null, verdict: null, run: null }] })); } const enabled = connected.filter((record) => repoPolicy(record).enabled); const [listed, held] = await Promise.all([listingOf(member, connected), heldPulls(tenantId)]); - const key = [deps.versions.get(tenantId), listed?.at, deps.ready(tenantId), Math.floor(deps.now().getTime() / REBUILD_MS), ...connected.map(triageStateKey), "--", ...enabled.map(triageStateKey), "--", ...held.map(pullKey).sort()].join("\n"); + const open = openNumbers(listed, enabled, held); + const pulls = [...open.values()].flatMap(({ repo, source, numbers }) => [...numbers].map((number) => ({ repo, source, number }))); + const live = livePulls(await deps.managerRuns(tenantId, pulls)); + const runs = [...live.values()].map((pull) => `${pullKey(pull)}:${pull.runs.map((run) => `${run.runId}:${run.status}`).join(",")}`).sort(); + const key = [listed?.at, Math.floor(deps.now().getTime() / REBUILD_MS), ...connected.map(triageStateKey), "--", ...enabled.map(triageStateKey), "--", ...held.map(pullKey).sort(), "--", ...runs].join("\n"); const hit = kept.get(tenantId); if (hit?.key === key) return respond(req, hit.body); - const body = bodyOf(await openList(tenantId, enabled, held, connected, listed, live)); + const body = bodyOf(await openList(tenantId, enabled, connected, open, live)); kept.set(tenantId, { key, body }); return respond(req, body); }; diff --git a/apps/hub/src/github/verdict-record-store.ts b/apps/hub/src/github/verdict-record-store.ts index cb9f3f591..f0ce5aaaa 100644 --- a/apps/hub/src/github/verdict-record-store.ts +++ b/apps/hub/src/github/verdict-record-store.ts @@ -63,7 +63,7 @@ export type ApprovedDo = { repo: string; number: number; effectId: string; statu const WRITTEN = new Set(["done", "satisfied"]); /** The verdict with each approved Do's outcome on its action; an approval that wrote it is not undone by one refused later. */ -export function withApproved(detail: VerdictDetail, approved: readonly ApprovedDo[]): VerdictDetail { +function withApproved(detail: VerdictDetail, approved: readonly ApprovedDo[]): VerdictDetail { if (!detail.actions || approved.length === 0) return detail; const actions = detail.actions.map(function outcomeOf(action) { return approved.reduce(function apply(current, { effectId, status, error }) { @@ -166,10 +166,6 @@ export function createVerdictRecordStore(db: DB["db"]) { }); }, - recordApproved(tenantId: string, approved: readonly ApprovedDo[]): Promise { - return recordApproved(db, tenantId, approved); - }, - /** The newest verdict of each pull request on each host. */ latest(tenantId: string, pulls: readonly PullRef[]): Promise { if (pulls.length === 0) return Promise.resolve([]); diff --git a/apps/hub/src/github/verdict-row.ts b/apps/hub/src/github/verdict-row.ts index 624fdd2d1..a00074b40 100644 --- a/apps/hub/src/github/verdict-row.ts +++ b/apps/hub/src/github/verdict-row.ts @@ -9,7 +9,7 @@ import type { RecordedDo, RecordedVerdict } from "./verdict-record-store.js"; export type Trigger = { repo: string; source: RepoSource; number: number; headSha: string; startedAt: string; event?: string }; export type Summary = Omit; -export type Ruled = { facts: Record; evidence: string[] }; +type Ruled = { facts: Record; evidence: string[] }; /** A webhook mail's `event:action`; catch-up and portal mails name none. */ export function eventOf(payload: Record): string | undefined { diff --git a/apps/hub/src/manager-state.ts b/apps/hub/src/manager-state.ts index 920343b22..b4f0ee9dd 100644 --- a/apps/hub/src/manager-state.ts +++ b/apps/hub/src/manager-state.ts @@ -400,10 +400,10 @@ export async function passRequest(db: DB["db"], tenantId: string, pr: PullReques } /** - * A pull request a manager claimed: its closed passes; while its manager is up, that run and the head of the newest pull - * request mail recorded; and the pass a manager that went away left open. + * A pull request a manager claimed: its closed passes; while its manager is up, that run, the head of the newest pull + * request mail recorded and the pass it has open; and the pass a manager that went away left open. */ -export type ManagedPull = { pr: PullRequestRef; history: ClosedPass[]; live?: { runId: string; head?: string }; gone?: { runId: string; pass: Pass } }; +export type ManagedPull = { pr: PullRequestRef; history: ClosedPass[]; live?: { runId: string; head?: string; pass?: Pass }; gone?: { runId: string; pass: Pass } }; /** The newest head a pull request mail named, which a live manager takes next or has taken. */ async function newestHead(tx: Tx, tenantId: string, pr: PullRequestRef): Promise { @@ -434,7 +434,7 @@ export async function managedPulls(db: DB["db"], tenantId: string, pulls: readon managed.push({ pr, history: claim.history, ...(claim.pass && { gone: { runId, pass: claim.pass } }) }); } else { const head = await newestHead(tx, tenantId, pr); - managed.push({ pr, history: claim.history, live: { runId, ...(head !== undefined && { head }) } }); + managed.push({ pr, history: claim.history, live: { runId, ...(head !== undefined && { head }), ...(claim.pass && { pass: claim.pass }) } }); } } return managed; diff --git a/apps/hub/src/server.test.ts b/apps/hub/src/server.test.ts index 18ac2684b..9095cdc7c 100644 --- a/apps/hub/src/server.test.ts +++ b/apps/hub/src/server.test.ts @@ -3,17 +3,18 @@ import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; import { type } from "arktype"; import { createHmac, generateKeyPairSync, randomBytes } from "node:crypto"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { and, count, eq, sql } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { createDB, schema } from "@intx/db"; import { WORKFLOW_RUN_REF, workflowRunRepoIdForAddress } from "@intx/hub-sessions"; import { formatRunAddress } from "@intx/types"; import { findArtifactByTitle, getArtifact } from "@corbits/artifacts"; -import { checkPackName, emptyPack, parseTriageState, stockTriggerMail, triageStateName, TriageStats } from "@corbits/triage-contracts"; +import { checkPackName, emptyPack, parseTriageState, triageStateName, TriageStats } from "@corbits/triage-contracts"; import type { ResolvedTarget, VerdictWord } from "@corbits/triage-contracts"; import { GITHUB_HOOK_HEADERS, xHubHookMetadata } from "@corbits/common/hook-metadata"; import { doEffectId } from "@corbits/triage-workflows/logic/actions"; import { workflow as prTriageWorkflow } from "@corbits/triage-workflows/pr-triage-github"; -import { verdictRecord } from "./github/verdict-record-store.js"; +import { replaceVerdict } from "./github/verdict-record-store.js"; +import { mirrorOutcomesOf, summaryOf } from "./github/verdict-row.js"; import { ensureTriageRole } from "./github/automation.js"; import { patchCorbitsTriage, upsertConnectedRepos } from "./github/tenant-config.js"; import { homedir, tmpdir } from "node:os"; @@ -581,17 +582,15 @@ const SAVED_PACK = { actions: { when: ["opened", "updated", "checks", "ready"], pass: [{ do: "labels", target: { from: "list", labels: ["checked"] } }], fail: [], unsure: [] }, }; -function timedRun(started: Date, tookMs: number, verdict: Record, mirrored = false): SeededEvent[] { - const at = (offsetMs: number) => new Date(started.getTime() + offsetMs).toISOString(); +/** A manager's verdict as its pass records it: made `tookMs` after the pass started, its comment posted when `mirrored`. */ +async function seedVerdict(db: Db, tenantId: string, runId: string, started: Date, tookMs: number, verdict: Record, mirrored = false): Promise { const number = Number(verdict["number"]); - const payload = stockTriggerMail({ kind: "pr", repo: REPO, prNumber: number, headSha: verdict["headSha"] }); - const mirror = mirrored ? { call: `mirror:${REPO.replace("/", "--")}:${number}`, ok: true } : { skipped: true }; - return [ - { type: "RunStarted", seq: 0, at: at(0), trigger: { type: "mail", payload } }, - { type: "StepCompleted", seq: 1, at: at(tookMs), stepId: "evaluate", output: { ref: `inline:${JSON.stringify(verdict)}` } }, - { type: "StepCompleted", seq: 2, at: at(tookMs + 1_000), stepId: "mirror", output: { ref: `inline:${JSON.stringify({ reply: JSON.stringify(mirror) })}` } }, - { type: "RunCompleted", seq: 3, at: at(tookMs + 2_000) }, - ]; + const headSha = String(verdict["headSha"]); + const trigger = { repo: REPO, source: "github" as const, number, headSha, startedAt: started.toISOString() }; + const mirror = mirrorOutcomesOf(mirrored ? { call: `mirror:${REPO.replace("/", "--")}:${number}`, ok: true } : undefined); + const summary = summaryOf(verdict, new Date(started.getTime() + tookMs).toISOString(), trigger, mirror, undefined); + if (!summary) throw new Error(`no summary for ${runId}`); + await replaceVerdict(db, tenantId, { runId, repo: REPO, number, headSha, startedAt: started, ...summary, runStatus: "completed" }); } function statsVerdict(number: number, headSha: string, fields: Record) { @@ -607,7 +606,7 @@ test("triage stats count a repository's recorded verdicts and Dos", async () => const dataDir = temporaryDirectory(); const { db, close } = createDB(DATABASE); try { - const origin = await startHub({ GITHUB_API_ORIGIN: github.origin, TRIAGE_RECONCILE_INTERVAL_MS: "200" }, dataDir); + const origin = await startHub({ GITHUB_API_ORIGIN: github.origin }, dataDir); const headers = await signedIn(origin); const tenantId = await createTenant(origin, headers); const otherTenantId = await createTenant(origin, headers); @@ -618,23 +617,11 @@ test("triage stats count a repository's recorded verdicts and Dos", async () => const ci = (result: string, check = "ci-green") => ({ check, kind: "machine", result, reason: "", evidence: [] }); // Noon UTC yesterday keeps the runs on one UTC day at any hour. const started = new Date(new Date().setUTCHours(-12, 0, 0, 0)); - await seedDeployment(db, dataDir, tenantId, [ - timedRun(started, 60_000, statsVerdict(7, "sha7", { checks: [ci("pass", "ci")], request: { repo: REPO, number: 7, labels: [], owned: [], comment: "Looks good" } }), true), - timedRun(new Date(started.getTime() + 600_000), 180_000, statsVerdict(8, HEAD, { - state: "awaiting-review", actor: "maintainer", checks: [ci("unconfirmed")], actions: [{ ...verdictStep(label, HEAD), status: "done" }, verdictStep(note, HEAD)], - })), - timedRun(new Date(started.getTime() + 1_200_000), 900_000, statsVerdict(8, HEAD, { state: "stale-unknown", actor: "system", degraded: "error", checks: [ci("fail")] })), - ], prTriageWorkflow.id); - - // Answers are cached for a minute, so the route is asked only once the reconciler recorded every verdict. - const deadline = Date.now() + 20_000; - let recorded = 0; - while (recorded < 3 && Date.now() < deadline) { - await Bun.sleep(200); - const [row] = await db.select({ rows: count() }).from(verdictRecord).where(eq(verdictRecord.tenantId, tenantId)); - recorded = row?.rows ?? 0; - } - expect(recorded).toBe(3); + await seedVerdict(db, tenantId, "wfr_stats_7", started, 60_000, statsVerdict(7, "sha7", { checks: [ci("pass", "ci")], request: { repo: REPO, number: 7, labels: [], owned: [], comment: "Looks good" } }), true); + await seedVerdict(db, tenantId, "wfr_stats_8", new Date(started.getTime() + 600_000), 180_000, statsVerdict(8, HEAD, { + state: "awaiting-review", actor: "maintainer", checks: [ci("unconfirmed")], actions: [{ ...verdictStep(label, HEAD), status: "done" }, verdictStep(note, HEAD)], + })); + await seedVerdict(db, tenantId, "wfr_stats_8b", new Date(started.getTime() + 1_200_000), 900_000, statsVerdict(8, HEAD, { state: "stale-unknown", actor: "system", degraded: "error", checks: [ci("fail")] })); async function stats(tenant: string) { const res = await fetch(`${origin}/api/integrations/triage-stats/${tenant}?repo=${REPO}`, { headers }); diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 155c2a61c..11b03819a 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -74,7 +74,7 @@ import { createProviderHookHandler } from "./provider-hooks/intake.js"; import { MAX_BODY_BYTES } from "./read-json.js"; import { createCoalescer, type Coalescer } from "./github/coalescer.js"; import { DeliveryCache } from "./github/dedupe.js"; -import { activeTenants, readyMaterializer, recentWorkers, systemSenderMaterializer, resolveLiveDeployments, sameSource, TriageStoppedError, withWorkflowVersions, type LiveDeployment } from "./github/deployment.js"; +import { readyMaterializer, systemSenderMaterializer, resolveLiveDeployments, sameSource, TriageStoppedError, type LiveDeployment } from "./github/deployment.js"; import { createGithubOpenPulls, createMemberOpenPulls, createOpenPullsCache, keptOpenPrs } from "./github/open-pulls.js"; import { createRepoReads } from "./github/repo-reads.js"; import { createGithubPrActions } from "./github/pr-actions.js"; @@ -88,8 +88,6 @@ import { createTriageStats } from "./github/triage-stats.js"; import { createHostReads, createOutcomeReader } from "./github/pr-outcomes.js"; import { createTriageTimeline } from "./github/triage-timeline.js"; import { createAuditLog } from "./audit-log.js"; -import { createRunRecorder } from "./github/run-recorder.js"; -import { createRecordPass, createTenantObserver, createTenantVersions } from "./github/tenant-observer.js"; import { createTriageVerdicts } from "./github/triage-verdicts.js"; import { createGithubPrDetails } from "./github/pr-details.js"; import { createMergeStates } from "./github/merge-states.js"; @@ -104,7 +102,6 @@ import { githubUsage } from "./github/github-usage.js"; import { createTriageReconciler, type ReconcileOutcome } from "./github/triage-reconciler.js"; import { createRepoActivity } from "./github/repo-activity.js"; import { createTenantGithub, createTenantHeadChecks, createTenantOpenHead, createTenantOpenHeads, createTenantOpenPrs } from "./github/tenant-open-heads.js"; -import { createSettledStatusReader, createTriageRuns } from "./github/triage-runs.js"; import { createTriageStateStore } from "./github/triage-state-store.js"; import { markBaseMoved, type TriageJob } from "./github/triage-queue.js"; import { createGithubManifestIntegration } from "./github/manifest.js"; @@ -140,19 +137,13 @@ const EXPECTED_DEPLOYMENT_SET = { }, }; -// Runs whose trigger and outcome the reconciler remembers between passes. -const TRIAGE_KNOWN_RUNS = 10_000; const TRIAGE_RETRY_MS = 15_000; const TRIAGE_RETRY_MAX_MS = 120_000; /** A repository is listed again once its listing is this old, and every pass once its webhook (the App's, on GitHub) heard nothing this long. */ const TRIAGE_RELIST_QUIET_MS = 30 * 60_000; /** A listing that started this soon after a repository's webhook is read again, since GitHub's listing can trail it. */ const TRIAGE_RELIST_GRACE_MS = 60_000; -/** How often the live logs of tenants with a pr-triage run in flight are read and recorded, and how stale a shared read may be. */ -const TRIAGE_RECORD_MS = 2_000; const GITHUB_CALLS_LOG_MS = 60_000; -/** A tenant whose logs could not be read is left at most this long before the recording loop tries again. */ -const TRIAGE_RECORD_MAX_BACKOFF_MS = 5 * 60_000; /** How often E2B sandboxes are swept and extended; well inside their one-hour timeout. */ const E2B_TEND_MS = 10 * 60_000; /** A GitHub token that could not be renewed is tried again within its remaining quarter hour. */ @@ -202,7 +193,6 @@ const sidecarProvisioners = [local.provisioner, ...workloadProvisioners]; let shuttingDown = false; let cronTicker: CronTicker | undefined; let triageLoop: ReturnType | undefined; -let recordLoop: ReturnType | undefined; let tokenLoop: ReturnType | undefined; let coalescer: Coalescer | undefined; let e2bTender: ReturnType | undefined; @@ -216,7 +206,6 @@ async function shutdown(): Promise { shuttingDown = true; cronTicker?.stop(); triageLoop?.stop(); - recordLoop?.stop(); tokenLoop?.stop(); coalescer?.stop(); clearInterval(e2bTender); @@ -533,9 +522,6 @@ function reconcileTenants() { } const launchSpecs = createWorkflowRunLaunchSpecStore(composition.db); const managerRuns = createManagerRuns(composition.db, verdictRecordStore, now); -async function liveTriageDeployments(tenantId: string, workflow: string) { - return withWorkflowVersions(launchSpecs, await resolveLiveDeployments(composition.db, tenantId, workflow)); -} const tenantOpenHeads = createTenantOpenHeads(tenantSources); const tenantOpenHead = createTenantOpenHead(tenantSources); const tenantHeadChecks = createTenantHeadChecks(tenantSources); @@ -546,59 +532,11 @@ const githubOpenPulls = createGithubOpenPulls({ authorize: authorizePortal, }); const triageStateStore = createTriageStateStore({ db: composition.db, writerFor: githubSenderPrincipal, log: logJson }); -const observeTriageRuns = createTriageRuns({ - runReader: composition.runReader, - readSettled: createSettledStatusReader(composition.db), - maxKnownRuns: TRIAGE_KNOWN_RUNS, -}); -const tenantVersions = createTenantVersions(); const reconcilePolicy = triageReconcilePolicy(env); -const runRecorder = createRunRecorder({ verdicts: verdictRecordStore, log: logJson }); - -/** - * The observer and recording loop that copy verdicts into the verdict table from the run logs of one workflow's - * listeners and of the workers of `workerIds`: a host's pr-triage reads its PR workflow variants' workers too, so their - * runs settle its heads. - */ -function recordingStack(workflowId: string, workerIds: readonly string[] = [workflowId]) { - function liveDeployments(tenantId: string) { - return liveTriageDeployments(tenantId, workflowId); - } - async function workerDeployments(tenantId: string, endedSince: Date) { - const workers = await Promise.all(workerIds.map((id) => recentWorkers(composition.db, tenantId, id, endedSince))); - return withWorkflowVersions(launchSpecs, workers.flat()); - } - async function activeTriageTenants(what: "runs" | "deployments", endedSince?: Date) { - const active = await Promise.all([...new Set([workflowId, ...workerIds])].map((id) => activeTenants(composition.db, id, what, endedSince))); - return [...new Map(active.flat().map((tenant) => [tenant.id, tenant])).values()]; - } - const observer = createTenantObserver({ - liveDeployments, - // Reconciler passes can be two intervals apart (a kick is skipped while one drains), and an observation is reused for a record interval. - workers: { deployments: workerDeployments, endedWithinMs: 2 * triageReconcileIntervalMs(env) + 2 * TRIAGE_RECORD_MS }, - observeRuns: observeTriageRuns, - recordRuns: runRecorder, - versions: tenantVersions, - minIntervalMs: TRIAGE_RECORD_MS, - now, - }); - const recordPass = createRecordPass({ - observer, - activeTenants: activeTriageTenants, - endedWithinMs: 2 * TRIAGE_RECORD_MS, - backoffMs: TRIAGE_RECORD_MS, - maxBackoffMs: TRIAGE_RECORD_MAX_BACKOFF_MS, - now, - log: logJson, - }); - return { observer, recordPass, liveDeployments }; -} -/** One host's observer, run recorder and reconciler. */ -function triageStack(triage: TriageWorkflow) { - const variants = PR_WORKFLOW_VARIANTS.filter((variant) => variant.source === triage.source).map((variant) => variant.id); - const { observer, recordPass } = recordingStack(triage.id, [triage.id, ...variants]); - const reconcile = createTriageReconciler({ +/** One host's reconciler. */ +function reconcilerFor(triage: TriageWorkflow) { + return createTriageReconciler({ source: triage.source, tenants: reconcileTenants, listener: liveListenerOf, @@ -606,7 +544,6 @@ function triageStack(triage: TriageWorkflow) { verdicts: verdictRecordStore, openHeadsFor: tenantOpenHeads, activity: repoActivity, - observeTenant: observer.observe, managerRuns, store: triageStateStore, readCheckPack, @@ -615,42 +552,18 @@ function triageStack(triage: TriageWorkflow) { now, log: logJson, }); - return { triage, observer, recordPass, reconcile }; } -const stacks = TRIAGE_WORKFLOWS.map(triageStack); -// pr-triage-historical is only recorded: the reconcilers never mail or rotate it. -// A PR workflow variant's listener runs only the approvals of Dos its workers held; those workers are its host's stack's. -const recordings = [...stacks, recordingStack(prTriageHistoricalWorkflow.id), ...PR_WORKFLOW_VARIANTS.map((variant) => recordingStack(variant.id, []))]; -// The portal's verdict listing reads every host's observations. -const tenantObserver = { - last(tenantId: string) { - const observations = recordings.flatMap((stack) => stack.observer.last(tenantId) ?? []); - if (observations.length === 0) return undefined; - return { - deployments: observations.flatMap((observation) => observation.deployments), - observed: observations.flatMap((observation) => observation.observed), - workers: observations.flatMap((observation) => observation.workers), - }; - }, -}; -function recordsReady(tenantId: string): boolean { - return recordings.every((stack) => stack.recordPass.ready(tenantId)); -} -async function recordAll() { - for (const stack of recordings) await stack.recordPass.pass(); - return { retry: false }; -} +const reconcilers = TRIAGE_WORKFLOWS.map(reconcilerFor); async function reconcileAll(): Promise { // A host removed through the stock credential route tells the hub nothing, so each pass releases what it left running. for (const tenant of await reconcileTenants()) await releaseHostsOf(tenant.id); const outcomes: ReconcileOutcome[] = []; - for (const stack of stacks) outcomes.push(await stack.reconcile()); + for (const reconcile of reconcilers) outcomes.push(await reconcile()); const retry = outcomes.some((outcome) => outcome.retry); const limits = outcomes.flatMap((outcome) => outcome.retryAt ?? []); return limits.length === 0 ? { retry } : { retry, retryAt: Math.min(...limits) }; } -recordLoop = createReconcileLoop(recordAll, { intervalMs: TRIAGE_RECORD_MS, retryMs: TRIAGE_RECORD_MS, retryMaxMs: TRIAGE_RECORD_MS }, logJson); triageLoop = createReconcileLoop(reconcileAll, { intervalMs: triageReconcileIntervalMs(env), retryMs: TRIAGE_RETRY_MS, retryMaxMs: TRIAGE_RETRY_MAX_MS }, logJson); const githubPrTriage = createGithubPrTriage({ ...credentialDeps, @@ -847,9 +760,7 @@ const integrations = createIntegrationsApp({ db: composition.db, getSession: com ...credentialDeps, authorize: authorizePortal, verdicts: verdictRecordStore, - observer: tenantObserver, - ready: recordsReady, - versions: tenantVersions, + managerRuns, openPulls: openPullsCache, list: tenantOpenPrs, triageState: triageStateStore, @@ -1001,7 +912,6 @@ async function routeRequest(req: Request, server: Parameters } triageLoop.start(); -recordLoop.start(); tokenLoop.start(); logJson({ diff --git a/apps/web/src/lib/backlog-status.test.ts b/apps/web/src/lib/backlog-status.test.ts index de60b26e9..f5f828df0 100644 --- a/apps/web/src/lib/backlog-status.test.ts +++ b/apps/web/src/lib/backlog-status.test.ts @@ -1,77 +1,13 @@ -import { describe, expect, test } from "bun:test"; -import { stockTriggerMail } from "@corbits/triage-contracts"; -import { catchingUpRepos, isRepoCatchingUp } from "./backlog-status.ts"; -import type { HubRun, RunLog } from "./hub-api.ts"; - -type Source = { logs?: RunLog[]; runs?: HubRun[] }; - -const catchingUp = (source: Source, repo: string) => isRepoCatchingUp(source.logs ?? [], source.runs ?? [], [], { name: repo }); - -const backlogRun = (status: HubRun["status"]): HubRun => ({ - id: "run-backlog", - definitionId: "def-backlog", - definitionName: "pr-triage-historical", - tenantId: "tnt", - address: "run-backlog", - status, - createdAt: "2026-03-10T00:00:00.000Z", - updatedAt: "2026-03-10T00:00:00.000Z", -}); - -const backlogLog = (repo: string, repoSource?: string, runId = "run-backlog"): RunLog => ({ - runId, - anchorRunId: runId, - events: [{ seq: 0, type: "RunStarted", body: { trigger: { payload: stockTriggerMail({ kind: "backlog", repo, ...(repoSource !== undefined && { repoSource }) }) } } }], -}); - -describe("isRepoCatchingUp", () => { - test("a live pr-triage-historical run for that repo is catching up", () => { - const source = { - runs: [backlogRun("running")], - logs: [backlogLog("acme/widgets")], - }; - expect(catchingUp(source, "acme/widgets")).toBe(true); - }); - - test("a running listener parent does not keep a completed child with no pull requests catching up", () => { - const source = { - runs: [ - { ...backlogRun("running"), id: "deploy-backlog" }, - { ...backlogRun("stopped"), id: "child-widgets", createdAt: "2026-03-10T00:01:00.000Z" }, - ], - logs: [ - { - runId: "child-widgets", - anchorRunId: "deploy-backlog", - events: [ - { - seq: 0, - type: "RunStarted", - body: { trigger: { payload: stockTriggerMail({ kind: "backlog", repo: "acme/widgets" }) } }, - }, - ], - }, - ], - }; - expect(catchingUp(source, "acme/widgets")).toBe(false); - }); - - test("a log not bound to a backlog run is not a live run", () => { - const source = { - logs: [ - { - runId: "run-other", - anchorRunId: "run-other", - events: [{ seq: 0, type: "RunStarted", body: { trigger: { payload: stockTriggerMail({ repo: "acme/widgets" }) } } }], - }, - ], - }; - expect(catchingUp(source, "acme/widgets")).toBe(false); - }); - - test("a repository is caught up only on the host whose pr-triage ran, though the other host has the same name", () => { - const runs = [backlogRun("running"), { ...backlogRun("running"), id: "run-bitbucket" }]; - const logs = [backlogLog("acme/widgets"), backlogLog("acme/widgets", "bitbucket-datacenter", "run-bitbucket")]; - expect(catchingUpRepos(logs, runs, [{ name: "acme/widgets", source: "github" }])).toEqual([{ name: "acme/widgets", source: "bitbucket-datacenter" }]); - }); +import { expect, test } from "bun:test"; +import type { TriageVerdicts } from "@corbits/triage-contracts"; +import { catchingUpRepos } from "./backlog-status.ts"; + +function pull(repo: string, source: "github" | "bitbucket-datacenter", status: "running" | "failed"): TriageVerdicts["pulls"][number] { + return { repo, number: 1, source, activityAt: null, verdict: null, run: { runId: "run_manager", startedAt: "2026-10-07T00:00:00.000Z", status } }; +} + +test("a repository is catching up while a pull request of it has a manager's run going and it has no verdict yet, on its own host", () => { + const pulls = [pull("acme/widgets", "github", "running"), pull("acme/widgets", "bitbucket-datacenter", "running"), pull("acme/gadgets", "github", "failed")]; + expect(catchingUpRepos(pulls, [])).toEqual([{ name: "acme/widgets", source: "github" }, { name: "acme/widgets", source: "bitbucket-datacenter" }]); + expect(catchingUpRepos(pulls, [{ name: "acme/widgets", source: "github" }])).toEqual([{ name: "acme/widgets", source: "bitbucket-datacenter" }]); }); diff --git a/apps/web/src/lib/backlog-status.ts b/apps/web/src/lib/backlog-status.ts index 2007a0bed..3028e94fd 100644 --- a/apps/web/src/lib/backlog-status.ts +++ b/apps/web/src/lib/backlog-status.ts @@ -1,41 +1,15 @@ -import { asRecord } from "@corbits/common"; -import { repoSourceOf, triggerRequestOf } from "@corbits/triage-contracts"; -import type { HubRun, RunLog } from "./hub-api.ts"; +import type { TriageVerdicts } from "@corbits/triage-contracts"; import { sameRepo, type RepoId } from "./repo-rows.ts"; -function eventPayload(body: unknown): Record { - const row = asRecord(body) ?? {}; - const trigger = asRecord(row.trigger) ?? {}; - return asRecord(triggerRequestOf(trigger.payload ?? row.payload ?? row.content)) ?? {}; -} - -function backlogRepoFromLog(log: RunLog): RepoId | null { - for (const event of log.events) { - const payload = eventPayload(event.body); - if (payload.kind === "backlog" && typeof payload.repo === "string") return { name: payload.repo, source: repoSourceOf(payload.repoSource) }; - } - return null; -} - -function isLiveStatus(status: string): boolean { - const normalized = status.toLowerCase(); - return normalized === "running" || normalized === "deployed" || normalized === "pending"; -} - /** - * Repositories with a live pr-triage-historical run, bound by `{ kind: "backlog", repo, repoSource }` - * on its own events, and no pr-triage run yet; `triaged` are the repositories that have one, each on its host. + * Repositories with no verdict yet whose pull requests have a manager's run going, each on its host; `triaged` are the + * repositories that have a verdict. */ -export function catchingUpRepos(logs: RunLog[], runs: HubRun[], triaged: readonly RepoId[]): RepoId[] { - const live = new Set(runs.filter((run) => isLiveStatus(run.status)).map((run) => run.id)); +export function catchingUpRepos(pulls: TriageVerdicts["pulls"], triaged: readonly RepoId[]): RepoId[] { const repos: RepoId[] = []; - for (const log of logs) { - const repo = backlogRepoFromLog(log); - if (repo !== null && live.has(log.runId) && ![...triaged, ...repos].some((row) => sameRepo(row, repo))) repos.push(repo); + for (const pull of pulls) { + const repo = { name: pull.repo, source: pull.source }; + if (pull.run?.status === "running" && ![...triaged, ...repos].some((row) => sameRepo(row, repo))) repos.push(repo); } return repos; } - -export function isRepoCatchingUp(logs: RunLog[], runs: HubRun[], triaged: readonly RepoId[], repo: RepoId): boolean { - return catchingUpRepos(logs, runs, triaged).some((row) => sameRepo(row, repo)); -} diff --git a/apps/web/src/lib/hub-api.test.ts b/apps/web/src/lib/hub-api.test.ts index 7f689f299..099c50122 100644 --- a/apps/web/src/lib/hub-api.test.ts +++ b/apps/web/src/lib/hub-api.test.ts @@ -148,18 +148,15 @@ describe("projectQueue from the hub's verdicts", () => { expect(items.map(({ key, draft }) => ({ key, draft }))).toEqual([{ key: "acme/widgets#8", draft: true }]); }); - test("a pull request's run in flight, ended early or not read yet shows, with or without a previous verdict", () => { + test("a pull request's run in flight or ended early shows, with or without a previous verdict", () => { const run = (status: "running" | "stuck" | "failed") => ({ runId: `run-${status}`, startedAt: "2026-10-01T01:30:00.000Z", status }); const pulls = [ pull(8, { verdict: decided.verdict, run: run("running"), activityAt: "2026-10-01T01:30:00.000Z" }), pull(9, { run: run("stuck") }), pull(10, { run: run("failed") }), - pull(11, { verdict: decided.verdict, run: { status: "unknown" } }), ]; expect(projectQueue(pulls, [], openPulls).map(({ key, state, running, failure, activityAt }) => ({ key, state, running, failure, activityAt }))).toEqual([ { key: "acme/widgets#8", state: "needs-decision", running: true, failure: null, activityAt: "2026-10-01T01:30:00.000Z" }, - // Until the hub read the live logs since it started, a run may be going, so writes wait as for one that is. - { key: "acme/widgets#11", state: "needs-decision", running: true, failure: null, activityAt: null }, { key: "acme/widgets#9", state: "new", running: false, failure: "stuck", activityAt: null }, { key: "acme/widgets#10", state: "new", running: false, failure: "failed", activityAt: null }, ]); diff --git a/apps/web/src/lib/hub-api.ts b/apps/web/src/lib/hub-api.ts index 9113aaaa3..42406d40e 100644 --- a/apps/web/src/lib/hub-api.ts +++ b/apps/web/src/lib/hub-api.ts @@ -1,17 +1,15 @@ import { ApiError, type Transport, - type WorkflowRunEvent, } from "@intx/hub-client"; import { asRecord, asRepo, compare } from "@corbits/common"; import { GITHUB_HOOK_HEADERS, xHubHookMetadata } from "@corbits/common/hook-metadata"; -import type { ApprovalResponse, CredentialResponse, GrantResponse, PrincipalResponse, RoleResponse, WorkflowRunResponse } from "@intx/types"; +import type { ApprovalResponse, CredentialResponse, GrantResponse, PrincipalResponse, RoleResponse } from "@intx/types"; import { AuditLog, BacklogStarted, CodeReviewPlace, PolicyExported, PolicyFileRead, PolicyRead, PolicyUpgrades, readCheckPack, repoPolicy, repoSourceOf, checkPackName, CODE_REVIEW_PROVIDERS, codeReviewProviderOf, currentRows, parseCheckOutput, ACTION_KINDS, RETIRED_CHECK_IDS, DeployedVersions, InviteRoles, InviteSent, SessionList, TriageStats, TriageVerdicts, type ActionKind, type ArtifactRef, type AuthorKind, type CheckOutput, type CheckPack, type CodeReviewProvider, type MergeRisk, type PolicyFileStatus, type RepoCheckFlags, type RepoPolicy, type RepoSource, type VerdictDetail, type VerdictWord } from "@corbits/triage-contracts"; import { createGrantBody, type CreateGrantInput } from "./grant-actions.ts"; const WORKSPACE_SLUG = (import.meta as { env?: Record }).env?.VITE_WORKSPACE_SLUG?.trim() || "corbits-triage"; -export const BACKLOG_WORKFLOW = "pr-triage-historical"; const GITHUB_PROVIDER = "github"; const GITHUB_PROVIDER_PLUGIN = "http"; export const GITHUB_CREDENTIAL_NAME = "github"; @@ -109,13 +107,6 @@ export function isPerson(principal: HubPrincipal): boolean { } export type HubRole = typeof RoleResponse.infer; export type HubApproval = typeof ApprovalResponse.infer; -export type HubRun = typeof WorkflowRunResponse.infer; - -export type RunLog = { - runId: string; - anchorRunId: string; - events: WorkflowRunEvent[]; -}; type SectionDenied = { repos: boolean; @@ -1215,10 +1206,6 @@ export async function listApprovals(transport: Transport, tenantId: string): Pro return listAll(transport, `/api/tenants/${tid}/approvals`); } -export async function listRuns(transport: Transport, tenantId: string): Promise { - const tid = enc(requireTenantId(tenantId)); - return listAll(transport, `/api/tenants/${tid}/workflows/runs`); -} async function readCodeReviewPlace(transport: Transport, tid: string): Promise { return CodeReviewPlace.assert(await transport.fetch("GET", `/api/integrations/code-review/${tid}/place`)); @@ -1462,16 +1449,15 @@ function isGithubWriteApproval(approval: HubApproval): boolean { /** A stuck run is a failure, not running, so its Dos stay enabled: running them again is the way out. */ function failureOf(run: PullRun | null): RunFailure | null { - return run === null || run.status === "running" || run.status === "unknown" || run.status === "waiting-ci" ? null : run.status; + return run === null || run.status === "running" || run.status === "waiting-ci" ? null : run.status; } function waitingFields(run: PullRun | null | undefined): Pick { return run?.status === "waiting-ci" ? { waitingForCi: true, waitingHeadSha: run.headSha } : { waitingForCi: false, waitingHeadSha: null }; } -/** A run the hub has not read yet may be going, so it holds writes like one that is. */ function isRunning(run: PullRun | null): boolean { - return run?.status === "running" || run?.status === "unknown"; + return run?.status === "running"; } function verdictItem(pull: PullTriage, verdict: PullVerdict): PrItem { diff --git a/apps/web/src/lib/persisted-queries.test.ts b/apps/web/src/lib/persisted-queries.test.ts index dbc83c0a2..367c494ff 100644 --- a/apps/web/src/lib/persisted-queries.test.ts +++ b/apps/web/src/lib/persisted-queries.test.ts @@ -9,23 +9,18 @@ function persistedKeys(client: QueryClient): unknown[] { return dehydrate(client, { shouldDehydrateQuery: isPersistedQuery }).queries.map((query) => query.queryKey); } -test("the open set, its verdicts and finished logs are kept; running logs and other reads are not", () => { +test("the open set, its verdicts and what the inbox handled are kept; other reads are not", () => { const client = new QueryClient(); client.setQueryData(["open-pulls", "t1"], OPEN); client.setQueryData(["triage-verdicts", "t1"], { pulls: [], triaged: [] }); - client.setQueryData(["run-ids", "t1", "a"], []); - client.setQueryData(["run-log", "t1", "a", "done"], { runId: "done", anchorRunId: "a", events: [{ type: "RunCompleted" }] }); - client.setQueryData(["run-log", "t1", "a", "live"], { runId: "live", anchorRunId: "a", events: [{ type: "RunStarted" }] }); client.setQueryData(["handled-pulls", "t1"], {}); client.setQueryData(["hidden-dos", "t1"], []); client.setQueryData(["approvals", "t1"], []); - client.setQueryData(["runs", "t1"], []); client.getQueryCache().build(client, { queryKey: ["open-pulls", "t2"] }); expect(persistedKeys(client)).toEqual([ ["open-pulls", "t1"], ["triage-verdicts", "t1"], - ["run-log", "t1", "a", "done"], ["handled-pulls", "t1"], ["hidden-dos", "t1"], ]); diff --git a/apps/web/src/lib/persisted-queries.ts b/apps/web/src/lib/persisted-queries.ts index 6e590d857..1f11bdc56 100644 --- a/apps/web/src/lib/persisted-queries.ts +++ b/apps/web/src/lib/persisted-queries.ts @@ -1,17 +1,16 @@ import type { Query } from "@tanstack/react-query"; import { isHiddenDosQuery } from "./held-dos.ts"; import { isHandledPullsQuery, isOpenPullsQuery } from "./open-pulls.ts"; -import { isFinishedRunLogQuery } from "./portal.tsx"; import { isPrTimelineQuery } from "./pr-timeline.ts"; import { isRepoStatsQuery } from "./repo-stats.ts"; import { isTriageVerdictsQuery } from "./triage-verdicts.ts"; /** - * Kept in IndexedDB across reloads: finished run logs, which never change; the pull requests acted on from the inbox and the Dos - * hidden there; and the last known open pull requests and their verdicts, so the inbox shows at once and refetches; the last - * known triage stats of each repository and readiness history of each pull request, so they show at once. Running logs are not kept. + * Kept in IndexedDB across reloads: the pull requests acted on from the inbox and the Dos hidden there; and the last known + * open pull requests and their verdicts, so the inbox shows at once and refetches; the last known triage stats of each + * repository and readiness history of each pull request, so they show at once. */ export function isPersistedQuery(query: Query): boolean { - if (isFinishedRunLogQuery(query) || isHandledPullsQuery(query) || isHiddenDosQuery(query)) return true; + if (isHandledPullsQuery(query) || isHiddenDosQuery(query)) return true; return query.state.data !== undefined && (isOpenPullsQuery(query) || isTriageVerdictsQuery(query) || isRepoStatsQuery(query) || isPrTimelineQuery(query)); } diff --git a/apps/web/src/lib/portal.tsx b/apps/web/src/lib/portal.tsx index b1e607fb9..ab97fddd7 100644 --- a/apps/web/src/lib/portal.tsx +++ b/apps/web/src/lib/portal.tsx @@ -1,12 +1,11 @@ import { createContext, useCallback, useContext, useEffect, useMemo, useRef, type ReactNode } from "react"; import { toast } from "@corbits/react-ui/ui/toast"; import { toast as lastingToast } from "sonner"; -import { useQuery, useQueryClient, type Query, type QueryClient } from "@tanstack/react-query"; +import { useQuery, useQueryClient, type QueryClient } from "@tanstack/react-query"; import { useLocation } from "react-router-dom"; import { errorText } from "@corbits/common"; import { useSession } from "./session.tsx"; import { hubConfigured } from "./hub-origin.ts"; -import { isTerminalRunEvents } from "@intx/hub-client"; import { ApiError, createHubTransport, createLeavingHubTransport } from "./hub-transport.ts"; import { ANTHROPIC_PROVIDER, @@ -30,7 +29,6 @@ import { type InvitedAccess, type PrGithubWriteInput, type PrGithubWriteResult, - type RunLog, type PortalSnapshot, type RepoRecord, type WorkspaceOption, @@ -95,11 +93,8 @@ const GITHUB_ACTION_DONE: Record = { }; const PORTAL_QUERY_KEY = ["portal"] as const; -export const RUN_IDS_QUERY_KEY = "run-ids"; -export const RUN_LOG_QUERY_KEY = "run-log"; export const TRIAGE_VERDICTS_QUERY_KEY = "triage-verdicts"; export const APPROVALS_QUERY_KEY = "approvals"; -export const RUNS_QUERY_KEY = "runs"; export const PRINCIPALS_QUERY_KEY = "principals"; export const ROLES_QUERY_KEY = "roles"; export const DEPLOYED_WORKFLOWS_QUERY_KEY = "deployed-workflows"; @@ -111,11 +106,8 @@ export const INVITE_PATH = "/invite/"; const REFRESH_QUERY_KEYS = [ [CHECK_PACK_INDEX_QUERY_KEY], PORTAL_QUERY_KEY, - [RUN_IDS_QUERY_KEY], - [RUN_LOG_QUERY_KEY], [TRIAGE_VERDICTS_QUERY_KEY], [APPROVALS_QUERY_KEY], - [RUNS_QUERY_KEY], [PRINCIPALS_QUERY_KEY], [ROLES_QUERY_KEY], [DEPLOYED_WORKFLOWS_QUERY_KEY], @@ -125,21 +117,8 @@ const REFRESH_QUERY_KEYS = [ [TRIAGE_RELEASE_QUERY_KEY], ]; -function isRunLog(value: unknown): value is RunLog { - return typeof value === "object" && value !== null && "events" in value && Array.isArray(value.events); -} - -export function isFinishedRunLogQuery(query: Query): boolean { - return query.queryKey[0] === RUN_LOG_QUERY_KEY && isRunLog(query.state.data) && isTerminalRunEvents(query.state.data.events); -} - -/** A refresh rereads everything except finished run logs. */ -function canChange(query: Query): boolean { - return !isFinishedRunLogQuery(query); -} - async function invalidateChangeable(queryClient: QueryClient, queryKeys: ReadonlyArray): Promise { - await Promise.all(queryKeys.map((queryKey) => queryClient.invalidateQueries({ queryKey, predicate: canChange }))); + await Promise.all(queryKeys.map((queryKey) => queryClient.invalidateQueries({ queryKey }))); } /** A 401 means the hub ended the session, so the local session is cleared too. */ diff --git a/apps/web/src/lib/run-logs.ts b/apps/web/src/lib/run-logs.ts deleted file mode 100644 index 8b784abcc..000000000 --- a/apps/web/src/lib/run-logs.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { queryOptions, useQueries, type UseQueryResult } from "@tanstack/react-query"; -import { ApiError, isTerminalRunEvents, listWorkflowRuns, readWorkflowRunEvents, type Transport } from "@intx/hub-client"; -import { BACKLOG_WORKFLOW, type RunLog } from "./hub-api.ts"; -import { createHubTransport } from "./hub-transport.ts"; -import { RUN_IDS_QUERY_KEY, RUN_LOG_QUERY_KEY, usePortal } from "./portal.tsx"; -import { useDeployedWorkflows } from "./tenant-entities.ts"; - -const RUN_LIST_REFRESH_MS = 10_000; -const LIVE_LOG_REFRESH_MS = 10_000; - -type RunRef = { anchorRunId: string; runId: string }; - -/** - * A deployment the caller cannot read, or one removed since it was listed, contributes no runs. - * The anchor run's own log only spawns children and never ends, so it is not read. - */ -async function runsOf(transport: Transport, tenantId: string, anchorRunId: string): Promise { - try { - const runIds = await listWorkflowRuns(transport, tenantId, anchorRunId); - return runIds.filter((runId) => runId !== anchorRunId).map((runId) => ({ anchorRunId, runId })); - } catch (cause) { - if (cause instanceof ApiError && (cause.status === 403 || cause.status === 404)) return []; - throw cause; - } -} - -function deploymentRunsQuery(transport: Transport, tenantId: string, anchorRunId: string) { - return queryOptions({ - queryKey: [RUN_IDS_QUERY_KEY, tenantId, anchorRunId], - queryFn: () => runsOf(transport, tenantId, anchorRunId), - refetchInterval: RUN_LIST_REFRESH_MS, - }); -} - -async function readLog(transport: Transport, tenantId: string, ref: RunRef): Promise { - const log = await readWorkflowRunEvents(transport, tenantId, ref.anchorRunId, ref.runId); - return { runId: log.runId, anchorRunId: ref.anchorRunId, events: log.events }; -} - -function isEnded(log: RunLog | undefined): boolean { - return log !== undefined && isTerminalRunEvents(log.events); -} - -function runLogQuery(transport: Transport, tenantId: string, ref: RunRef) { - return queryOptions({ - queryKey: [RUN_LOG_QUERY_KEY, tenantId, ref.anchorRunId, ref.runId], - queryFn: () => readLog(transport, tenantId, ref), - // A finished run's log never changes again, so it is read once and kept. - staleTime: (query) => (isEnded(query.state.data) ? Infinity : LIVE_LOG_REFRESH_MS), - refetchInterval: (query) => (isEnded(query.state.data) ? false : LIVE_LOG_REFRESH_MS), - gcTime: Infinity, - }); -} - -function loaded(results: Array>): T[] { - return results.flatMap((result) => (result.data === undefined ? [] : [result.data])); -} - -/** The logs of the backlog workflow's live deployments' runs, which say which repositories are catching up; no other list reads run logs. */ -export function useBacklogLogs(): RunLog[] { - const { snapshot } = usePortal(); - const tenantId = snapshot?.workspace.tenantId; - const transport = createHubTransport(); - const deployed = useDeployedWorkflows(); - const anchors = deployed.rows.filter((deployment) => deployment.workflow === BACKLOG_WORKFLOW).map((deployment) => deployment.id); - const refs = useQueries({ queries: tenantId === undefined ? [] : anchors.map((anchor) => deploymentRunsQuery(transport, tenantId, anchor)), combine: loaded }); - return useQueries({ queries: tenantId === undefined ? [] : refs.flat().map((ref) => runLogQuery(transport, tenantId, ref)), combine: loaded }); -} diff --git a/apps/web/src/lib/tenant-entities.ts b/apps/web/src/lib/tenant-entities.ts index cc028d6b4..ffb7658f7 100644 --- a/apps/web/src/lib/tenant-entities.ts +++ b/apps/web/src/lib/tenant-entities.ts @@ -6,12 +6,10 @@ import { listDeployedWorkflows, listPrincipals, listRoles, - listRuns, type DeployedWorkflow, type HubApproval, type HubPrincipal, type HubRole, - type HubRun, } from "./hub-api.ts"; import { createHubTransport } from "./hub-transport.ts"; import { @@ -20,7 +18,6 @@ import { DEPLOYED_WORKFLOWS_QUERY_KEY, PRINCIPALS_QUERY_KEY, ROLES_QUERY_KEY, - RUNS_QUERY_KEY, usePortal, useSignOutWhenRejected, } from "./portal.tsx"; @@ -77,10 +74,6 @@ export function useApprovals(): TenantSection { return useTenantSection(APPROVALS_QUERY_KEY, listApprovals, LIVE_REFRESH_MS); } -export function useRuns(): TenantSection { - return useTenantSection(RUNS_QUERY_KEY, listRuns, LIVE_REFRESH_MS); -} - export function usePrincipals(): TenantSection { return useTenantSection(PRINCIPALS_QUERY_KEY, listPrincipals, DIRECTORY_REFRESH_MS); } diff --git a/apps/web/src/pages/Connect.tsx b/apps/web/src/pages/Connect.tsx index 080f8f294..09f9c26de 100644 --- a/apps/web/src/pages/Connect.tsx +++ b/apps/web/src/pages/Connect.tsx @@ -3,7 +3,7 @@ import { Badge, Button, StatusDot } from "@corbits/react-ui"; import { useNavigate } from "react-router-dom"; import { Check, CircleCheck, Sparkles, Upload } from "lucide-react"; import { errorText } from "@corbits/common"; -import { isRepoCatchingUp } from "../lib/backlog-status.ts"; +import { catchingUpRepos } from "../lib/backlog-status.ts"; import { githubAppSlugFromCredentials, hasActiveGithubCredential, portalConnected, type PortalSnapshot, type RepoRecord } from "../lib/hub-api.ts"; import { hubOnLocalhost } from "../lib/hub-origin.ts"; import { createHubTransport } from "../lib/hub-transport.ts"; @@ -20,11 +20,9 @@ import { import { useGithubSync } from "../lib/github-sync.ts"; import { counted } from "../lib/plural.ts"; import { usePortal } from "../lib/portal.tsx"; -import { hostedRepos } from "../lib/repo-rows.ts"; -import { useBacklogLogs } from "../lib/run-logs.ts"; +import { hostedRepos, sameRepo } from "../lib/repo-rows.ts"; import { useTriageVerdicts } from "../lib/triage-verdicts.ts"; import { UPGRADE_STEP_LABEL, useMayUpgrade, useTriageUpgrade, type UpgradeStep } from "../lib/triage-upgrade.ts"; -import { useRuns } from "../lib/tenant-entities.ts"; import { CopyField } from "../components/CopyField.tsx"; import { BitbucketDatacenterSource } from "../components/settings/BitbucketDatacenterSource.tsx"; import { DecisionModelForm } from "../components/DecisionModelForm.tsx"; @@ -71,10 +69,9 @@ export default function Connect() { const [secretNotice, setSecretNotice] = useState(""); const webhookUrl = snapshot ? githubWebhookUrl(snapshot.workspace.tenantId) : ""; const repos = snapshot?.repos ?? []; - const logs = useBacklogLogs(); const verdicts = useTriageVerdicts(); - const runs = useRuns(); const triaged = useMemo(() => hostedRepos(verdicts.data?.triaged ?? []), [verdicts.data]); + const catching = useMemo(() => catchingUpRepos(verdicts.data?.pulls ?? [], triaged), [verdicts.data, triaged]); const modelStored = snapshot?.decisionModel !== undefined; const mayUpgrade = useMayUpgrade(); const [upgradeStep, setUpgradeStep] = useState(null); @@ -216,7 +213,7 @@ export default function Connect() { } function renderInstallRow(repo: RepoRecord) { - const catchingUp = isRepoCatchingUp(logs, runs.rows, triaged, repo); + const catchingUp = catching.some((row) => sameRepo(row, repo)); return
{repo.name}{catchingUp ? Catching up open pull requests… : Connected}
; } diff --git a/apps/web/src/pages/Repositories.tsx b/apps/web/src/pages/Repositories.tsx index 1f4f38bab..b1b39d68a 100644 --- a/apps/web/src/pages/Repositories.tsx +++ b/apps/web/src/pages/Repositories.tsx @@ -19,9 +19,7 @@ import { usePortal } from "../lib/portal.tsx"; import { hostName } from "../lib/pull-link.ts"; import { useHubRepoPackStore } from "../lib/repo-pack.ts"; import { findRepo, hostedRepos, repoRows, selectedRepo, type RepoRow } from "../lib/repo-rows.ts"; -import { useBacklogLogs } from "../lib/run-logs.ts"; import { useTriageVerdicts } from "../lib/triage-verdicts.ts"; -import { useRuns } from "../lib/tenant-entities.ts"; import { relativeTime } from "../lib/triage-view.ts"; function plural(n: number, word: string): string { @@ -62,9 +60,7 @@ function isPlainClick(event: MouseEvent): boolean { export default function Repositories() { const { snapshot, readOnly } = usePortal(); const packStore = useHubRepoPackStore(); - const logs = useBacklogLogs(); const verdicts = useTriageVerdicts(); - const runs = useRuns(); const navigate = useNavigate(); const params = useParams(); const [search] = useSearchParams(); @@ -82,7 +78,7 @@ export default function Repositories() { const upgrades = usePolicyUpgrades(); const triaged = useMemo(() => hostedRepos(verdicts.data?.triaged ?? []), [verdicts.data]); const active = useMemo(() => hostedRepos(verdicts.data?.active ?? []), [verdicts.data]); - const catchingUp = useMemo(() => catchingUpRepos(logs, runs.rows, triaged), [logs, runs.rows, triaged]); + const catchingUp = useMemo(() => catchingUpRepos(verdicts.data?.pulls ?? [], triaged), [verdicts.data, triaged]); const rows = useMemo( () => repoRows(repos, packs, items, active, catchingUp, openPulls, pullsError), [repos, packs, items, active, catchingUp, openPulls, pullsError], diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c0082e577..66a3b9cdb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -73,7 +73,8 @@ GitHub ──webhook──► Hub ──spawn──► Sidecar ──► pr-tria mirrors the waiting marker. - **Reconciler**: one per host, for workspaces whose listener for the host is live. On an interval it compares each enabled repository's open pull - requests with the managers' runs, the run logs and stored state, and + requests with the managers' runs, their recorded verdicts and stored + state, and re-queues every head without a verdict, and every stale head at most once per 15 minutes, under a per-tenant in-flight cap. A repository's open pull requests are listed again only once a webhook named it, until a minute @@ -90,8 +91,6 @@ GitHub ──webhook──► Hub ──spawn──► Sidecar ──► pr-tria within the last two minutes; one deployed without a sidecar past that is released, so the next Upgrade deploys it again. The workflow-versions route reports each deployment's `live` the same way, which Settings reads. - - **Run recorder**: reads the live logs of tenants with a run in flight and - records each verdict and executed action. - **App Manifest conversion**: GitHub returns a one-time code that must be exchanged server side, so the private key never reaches the browser. The `state` round-tripped through GitHub is signed by the hub and bound to the @@ -414,8 +413,7 @@ label off that head, so it never stands on a head not yet triaged. is added, edited, moved or copied. The extension leaves code-pattern rules out and swaps the native module for a stub. 3. **route**: skips the judge when no question needs asking. -4. **judge**: one System One call per request of questions that no carried - answer matches. Each first-round request and the verdict request also asks +4. **judge**: one System One call per request of questions. Each first-round request and the verdict request also asks whether the pull request's text, diff or docs try to direct the judge; a confident yes in any of them leaves every judged rule unconfirmed and the verdict Unsure, whatever the verdict question answers. The only step that infers; see diff --git a/docs/DECISION_MODEL.md b/docs/DECISION_MODEL.md index ae8e148cb..2ad71b66b 100644 --- a/docs/DECISION_MODEL.md +++ b/docs/DECISION_MODEL.md @@ -21,7 +21,7 @@ Triage reaches the model through Interchange inference. The agents name the tena | OpenCode Zen | `https://opencode.ai/zen/v1` | `jev-1.13` | | Custom | any | any | -The judge eval pins OpenCode Zen `jev-1.13`. The judge is the only step that infers: a `map` over the rules step's chunks, one System One call per chunk, then a second `map` over focused's follow-up chunks and the verdict question (`triage-steps.ts`, `directors.ts`). A pull request without judged rules to ask, or whose judged rules an earlier run answered, asks the verdict question in the first map. A failed hard rule, a draft, a closed pull request or a policy with no decision-model rule on asks nothing. The request body is `{ state, questions, model }`; no system prompt reaches the model. +The judge eval pins OpenCode Zen `jev-1.13`. The judge is the only step that infers: a `map` over the rules step's chunks, one System One call per chunk, then a second `map` over focused's follow-up chunks and the verdict question (`triage-steps.ts`, `directors.ts`). A pull request without judged rules to ask asks the verdict question in the first map. A failed hard rule, a draft, a closed pull request or a policy with no decision-model rule on asks nothing. The request body is `{ state, questions, model }`; no system prompt reaches the model. ## Why Triage uses it @@ -113,17 +113,12 @@ Two different numbers come out of the answers: ## Reused answers -A re-check of a head the judge already answered, such as after a merge moved -the base branch, does not ask the model again for an unchanged request. Each -verdict keeps the answers it got, keyed by a SHA-256 of the judge's system -prompt and the exact request (`judgeKey`, `logic/quality.ts`). The reconciler -carries them in its catch-up mail only to a deployment on the same workflow -version and decision-model offering, and the facts step accepts them only from -the hub's own triage sender. Rules sends the judge only requests no carried -answer matches; changed content or docs are asked again. - -Only well-formed answers clear of the unsure band are kept, up to 16 KB per -verdict (`MAX_JUDGED_BYTES`, `actions/evaluate.ts`); the rest are asked again. +A pull request's manager keeps the judge's verdict for each head. A later pass +on the same head, such as a re-check after a merge moved the base branch, reuses +that verdict instead of asking again (`judgeScope`, +`packages/triage-manager/src/actions/judge.ts`). The judge is asked again for that +head only while the host's checks could still change its verdict: a host check +left unconfirmed, or an action held for CI. A new head is judged afresh. ## Cost per pull request diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index fe48d0597..b5128f04a 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -38,9 +38,9 @@ with the `intx-src` export condition, so every hub command runs with | `src/github/coalescer.ts` | Folds a pull request's webhook burst into one mail. | | `src/github/triage-reconciler.ts`, `reconcile-plan.ts`, `reconcile-loop.ts`, `triage-queue.ts` | Re-queues untriaged and stale heads through the listener. | | `src/github/triage-state-store.ts` | The per-repository triage state artifact. | -| `src/github/tenant-observer.ts`, `run-recorder.ts`, `triage-runs.ts` | Reads run logs and records verdicts. | +| `src/github/manager-runs.ts`, `triage-runs.ts` | The managers' runs of each open pull request as the reconciler and the verdicts route read them; the verdict and approval in a run's log, for held Dos and the audit log. | | `src/github/verdict-record-store.ts`, `triage-verdicts.ts`, `triage-stats.ts`, `triage-timeline.ts`, `pr-outcomes.ts` | Verdict records and the routes that serve them, including outcomes before and after Triage. Outcomes read at most the 50 newest-created pull requests on GitHub, or the newest page holding any (at most 8 pages of 50) on Bitbucket Data Center, with at most 8 reads at once, cached per pull request and per UTC day. | -| `src/github/backlog-triage.ts` | Mails `pr-triage-historical` to triage a repository's open pull requests. | +| `src/github/backlog-triage.ts` | Mails the host's listener a backlog to triage a repository's open pull requests. | | `src/github/manifest.ts` | GitHub App Manifest flow; setup state is an HMAC token keyed from `BETTER_AUTH_SECRET`, valid 10 minutes. | | `src/github/pr-*.ts`, `held-dos.ts` | Pull request actions, details and single-PR triage; approving a held action. | | `src/github/github-app-credential-adapter.ts` | The hub's GitHub fetch: App JWT to installation token, cached per installation. | diff --git a/packages/triage-contracts/src/hub-responses.ts b/packages/triage-contracts/src/hub-responses.ts index 94ae53040..cfb691246 100644 --- a/packages/triage-contracts/src/hub-responses.ts +++ b/packages/triage-contracts/src/hub-responses.ts @@ -81,7 +81,6 @@ const Verdict = type({ /** * The pull request's latest run, when it has not completed: still going, ended early, or running longer than any run can. - * `unknown` until the hub read the workspace's live logs since it started, when a run may be going unseen. * `waiting-ci`: no run yet, as the policy reads CI and it is still running on `headSha`, seen `since`. */ const Run = type({ @@ -89,7 +88,6 @@ const Run = type({ startedAt: "string", status: "'running' | 'failed' | 'cancelled' | 'stuck'", }) - .or({ status: "'unknown'" }) .or({ status: "'waiting-ci'", since: "string", headSha: "string" }); const Source = type.enumerated(...REPO_SOURCES);