diff --git a/apps/hub/src/bitbucket-datacenter/hook.test.ts b/apps/hub/src/bitbucket-datacenter/hook.test.ts index abaad799f..ce60857e1 100644 --- a/apps/hub/src/bitbucket-datacenter/hook.test.ts +++ b/apps/hub/src/bitbucket-datacenter/hook.test.ts @@ -67,7 +67,6 @@ function harness(overrides: Partial = {}) { } as never, cipher: { async decrypt() { return SECRET; } } as never, cache: new DeliveryCache(), - codeReview: { async offer() {} }, coalescer: { async accept(input) { accepted.push(input); diff --git a/apps/hub/src/bitbucket-datacenter/integration.ts b/apps/hub/src/bitbucket-datacenter/integration.ts index f3d09a0b9..504e7dfb3 100644 --- a/apps/hub/src/bitbucket-datacenter/integration.ts +++ b/apps/hub/src/bitbucket-datacenter/integration.ts @@ -59,7 +59,7 @@ type BitbucketDeps = PortalCredentialDeps & { egressAllowlist?: readonly string[]; /** Called once a repository is enabled, so its open pull requests are triaged without waiting for the next pass. */ onEnabled: (tenantId: string) => void; - /** Called after connect and disconnect: the deployed pr-triage entry follows the connected hosts. */ + /** Called after connect and disconnect, so a disconnected host's listener and managers are released. */ onConnectionChanged: (tenantId: string) => Promise; now: () => Date; log: (entry: Record) => void; diff --git a/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.test.ts b/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.test.ts index da6127545..966d548a0 100644 --- a/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.test.ts +++ b/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.test.ts @@ -1,41 +1,27 @@ import { expect, test } from "bun:test"; import type { DB } from "@intx/db"; import type { CredentialCipher } from "@intx/types"; -import type { TriageJob } from "../github/coordinator.js"; -import { jobWorkflowsFor, triageWorkflowFor } from "../github/source-workflow.js"; -import { checkedBitbucketSpawner, createTenantBitbucket } from "./tenant-bitbucket.js"; +import { checkedBitbucketDelivery, createTenantBitbucket } from "./tenant-bitbucket.js"; const TENANT = { config: { corbitsTriage: { bitbucketDatacenter: { baseUrl: "https://git.example" } } } }; -const KNOWN_BITBUCKET_WORKFLOWS = ["pr-triage-bitbucket-datacenter", "pr-code-review-bitbucket-datacenter", "pr-workflow-anthropic-bitbucket-datacenter"]; - -function job(workflow: string): TriageJob { - return { tenantId: "tnt_spawn", workflow, repo: "ACME/widgets", number: 1, headSha: "abc", source: "event", payload: {} }; -} - -test("no Bitbucket worker is spawned once the stored host resolves to an address the token may not reach", async () => { +test("no Bitbucket mail reaches the listener once the stored host resolves to an address the token may not reach", async () => { const tenantBitbucket = createTenantBitbucket({ db: { query: { tenant: { async findFirst() { return TENANT; } }, - credential: { async findFirst() { return { id: "crd_bb", tenantId: "tnt_spawn", status: "active", principalId: null }; } }, + credential: { async findFirst() { return { id: "crd_bb", tenantId: "tnt_mail", status: "active", principalId: null }; } }, }, } as unknown as DB["db"], cipher: {} as CredentialCipher, fetchImpl: fetch, async lookup() { return [{ address: "169.254.169.254", family: 4 }]; }, }); - const spawned: string[] = []; - const spawner = checkedBitbucketSpawner({ - async spawn(next) { spawned.push(next.workflow); return "run_worker"; }, - async ended() {}, - async release() {}, + const delivered: unknown[] = []; + const deliver = checkedBitbucketDelivery(async function record(_tenantId, _address, payload) { + delivered.push(payload); }, tenantBitbucket); - const workflows = jobWorkflowsFor("bitbucket-datacenter"); - expect(workflows).toEqual(expect.arrayContaining(KNOWN_BITBUCKET_WORKFLOWS)); - for (const workflow of workflows) { - await expect(spawner.spawn(job(workflow))).rejects.toThrow("host is refused"); - } - expect(await spawner.spawn(job(triageWorkflowFor("github").id))).toBe("run_worker"); - expect(spawned).toEqual([triageWorkflowFor("github").id]); + await expect(deliver("tnt_mail", "run_listener@tenant.example", { repoSource: "bitbucket-datacenter", repo: "ACME/widgets" })).rejects.toThrow("host is refused"); + await deliver("tnt_mail", "run_listener@tenant.example", { repoSource: "github", repo: "acme/widgets" }); + expect(delivered).toEqual([{ repoSource: "github", repo: "acme/widgets" }]); }); diff --git a/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.ts b/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.ts index bd6a4c9da..cb24753fa 100644 --- a/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.ts +++ b/apps/hub/src/bitbucket-datacenter/tenant-bitbucket.ts @@ -3,11 +3,11 @@ import type { DB } from "@intx/db"; import type { CredentialCipher } from "@intx/types"; import type { BitbucketFetch } from "@corbits/bitbucket-datacenter-reads"; +import { repoSourceOf } from "@corbits/triage-contracts"; +import { asRecord } from "@corbits/common"; import { tenantConfigOf } from "../github/bridge.js"; import { activeCredential, credentialSecret } from "../github/portal-credential.js"; import { triageNs } from "../github/tenant-config.js"; -import type { TriageJob, WorkerSpawner } from "../github/coordinator.js"; -import { jobWorkflowsFor } from "../github/source-workflow.js"; import { bitbucketFetch, lookupAddresses, resolveInstance, type AddressLookup } from "./client.js"; import { bitbucketConnection, TOKEN_CREDENTIAL } from "./records.js"; @@ -36,15 +36,14 @@ export function createTenantBitbucket(deps: TenantBitbucketDeps): TenantBitbucke }; } -const BITBUCKET_WORKFLOWS = new Set(jobWorkflowsFor("bitbucket-datacenter")); +export type Deliver = (tenantId: string, address: string, payload: unknown) => Promise; -/** A Bitbucket run sends the token to the stored instance, so its worker is spawned only while that host passes the check. */ -export function checkedBitbucketSpawner(spawner: WorkerSpawner, tenantBitbucket: TenantBitbucket): WorkerSpawner { - async function spawn(job: TriageJob): Promise { - if (BITBUCKET_WORKFLOWS.has(job.workflow) && (await tenantBitbucket(job.tenantId)) === undefined) { +/** A Bitbucket run sends the token to the stored instance, so its mail goes out only while that host passes the check. */ +export function checkedBitbucketDelivery(deliver: Deliver, tenantBitbucket: TenantBitbucket): Deliver { + return async function deliverChecked(tenantId, address, payload) { + if (repoSourceOf(asRecord(payload)?.["repoSource"]) === "bitbucket-datacenter" && (await tenantBitbucket(tenantId)) === undefined) { throw new Error("Bitbucket is not connected, has no active token, or its host is refused"); } - return spawner.spawn(job); - } - return { ...spawner, spawn }; + await deliver(tenantId, address, payload); + }; } diff --git a/apps/hub/src/code-review-runs.postgres.test.ts b/apps/hub/src/code-review-runs.postgres.test.ts index 7b5e3a693..8696f9f1d 100644 --- a/apps/hub/src/code-review-runs.postgres.test.ts +++ b/apps/hub/src/code-review-runs.postgres.test.ts @@ -3,7 +3,7 @@ import { expect, test } from "bun:test"; import { randomBytes } from "node:crypto"; import { sql } from "drizzle-orm"; import { createDB, runMigrations, schema } from "@intx/db"; -import { diffHash, type PullRequestToReview } from "./code-review-feed.js"; +import { diffHash, type PullRequestToReview } from "./pull-request-reader.js"; import { createCodeReviewRuns } from "./code-review-runs.js"; import { recordEvent, type PullRequestRef } from "./manager-state.js"; import { migrateTriageTables } from "./triage-db.js"; @@ -100,26 +100,27 @@ test("PostgreSQL: a pull request's latest code review is what its manager record // A finished review names its diff's hash, so a later head with the same diff takes it over. await recordEvent(db, TENANT, GITHUB, "run_live", { type: "reviewer-result", head: "sha4", payload: { kind: "code-review", result: OUTPUT, diffHash: diffHash(DIFF) } }); const hashed = { ...finished, headSha: "sha4", diffHash: diffHash(DIFF) }; - expect(await runs.carries(TENANT, RECORD, 7, head)).toBe(true); expect(await reviewsOf(GITHUB, "sha5")).toEqual({ review: hashed, completed: hashed, carriedTo: "sha5" }); - expect(hostReads).toBe(0); + expect(await reviewsOf(GITHUB, "sha5")).toMatchObject({ carriedTo: "sha5" }); + expect(hostReads).toBe(1); expect(await reviewsOf(GITHUB, "sha4")).toMatchObject({ carriedTo: null }); - expect(await runs.carries(TENANT, RECORD, 7, { ...head, headSha: "sha6", diff: BASE_EDITED })).toBe(false); - expect(await runs.carries(TENANT, RECORD, 7, { ...head, headSha: "sha7", diff: REBUILT })).toBe(false); - expect(await runs.carries(TENANT, RECORD, 7, { ...head, headSha: "sha8", diff: `${DIFF}+c\n` })).toBe(false); + for (const [headSha, diff] of [["sha6", BASE_EDITED], ["sha7", REBUILT], ["sha8", `${DIFF}+c\n`]] as const) { + head = { ...head, headSha, diff }; + expect(await reviewsOf(GITHUB, headSha)).toMatchObject({ carriedTo: null }); + } - head = { ...head, headSha: "sha10" }; + head = { ...head, headSha: "sha10", diff: DIFF }; expect(await reviewsOf(GITHUB, "sha10")).toMatchObject({ carriedTo: "sha10" }); expect(await reviewsOf(GITHUB, "sha11")).toMatchObject({ carriedTo: null }); - expect(hostReads).toBe(2); + expect(hostReads).toBe(6); // A read that fails, or finds another head, is not tried again on every poll. hostDown = true; expect(await reviewsOf(GITHUB, "sha12")).toMatchObject({ carriedTo: null }); expect(await reviewsOf(GITHUB, "sha12")).toMatchObject({ carriedTo: null }); expect(await reviewsOf(GITHUB, "sha11")).toMatchObject({ carriedTo: null }); - expect(hostReads).toBe(3); + expect(hostReads).toBe(7); } finally { await handle?.close(); await admin.db.execute(sql.raw(`drop database if exists ${database} with (force)`)); diff --git a/apps/hub/src/code-review-runs.ts b/apps/hub/src/code-review-runs.ts index bd8c743ea..fe9582002 100644 --- a/apps/hub/src/code-review-runs.ts +++ b/apps/hub/src/code-review-runs.ts @@ -5,7 +5,7 @@ import { eq } from "drizzle-orm"; import { type } from "arktype"; import { schema, type DB } from "@intx/db"; import { CodeReviewComment, DiffCut, REPO_SOURCES, repoSourceOf, StageResult, type RepoSource } from "@corbits/triage-contracts"; -import { diffHash, type PullRequestToReview } from "./code-review-feed.js"; +import { diffHash, type PullRequestToReview } from "./pull-request-reader.js"; import { repoRecords, sourceOf, triageNs, type RepoRecord } from "./github/tenant-config.js"; import type { Member } from "./integrations/app.js"; import { codeReviews, type CodeReviewRecord } from "./manager-state.js"; @@ -132,22 +132,11 @@ export function createCodeReviewRuns(deps: CodeReviewRunsDeps) { return { ...reviews, carriedTo: await carriedTo(tenantId, tenant.config, query, reviews) }; } - /** Whether `pull`'s head takes the newest review over rather than being reviewed; false when that can't be told. */ - async function carries(tenantId: string, record: RepoRecord, number: number, pull: PullRequestToReview): Promise { - try { - const completed = carriable(await reviewsOf(tenantId, sourceOf(record), record.name, number), pull.headSha); - return completed !== undefined && sameDiff(tenantId, record, number, completed, pull); - } catch (err) { - deps.log({ level: "warn", msg: "code_review_carry_unread", tenantId, repo: record.name, pr: number, error: String(err) }); - return false; - } - } - async function latestCodeReview(query: typeof CodeReviewQuery.infer, { tenantId }: Member): Promise { return Response.json(await latest(tenantId, query)); } - return { latest, carries, latestCodeReview }; + return { latest, latestCodeReview }; } export type CodeReviewRuns = ReturnType; diff --git a/apps/hub/src/code-review.postgres.test.ts b/apps/hub/src/code-review.postgres.test.ts index 7bc00fde7..bb5f8c78a 100644 --- a/apps/hub/src/code-review.postgres.test.ts +++ b/apps/hub/src/code-review.postgres.test.ts @@ -9,7 +9,6 @@ import { DOCKER_PLACE, DOCKER_REMOTE_PLACE } from "@corbits/provisioner-docker"; import { emptyPack, type CodeReviewPlace } from "@corbits/triage-contracts"; import { createCodeReviewPlace, createCodeReviewPlacement, createCodeReviewRun, createCodeReviewSwitch } from "./code-review.js"; import { NoLiveDeploymentError } from "./github/deployment.js"; -import { codeReviewSources } from "./github/source-workflow.js"; import { repoRecords, triageNs } from "./github/tenant-config.js"; const TEST_DB = { host: "localhost", port: 5432, user: "postgres", password: "postgres", database: "interchange" }; @@ -94,10 +93,8 @@ test("PostgreSQL: code review turns on only with an isolated place to run, the h expect((await answer({ repo: "acme/missing", enabled: true })).status).toBe(404); expect((await answer({ repo: "acme/gone", enabled: true })).body.error?.code).toBe("repo_not_connected"); expect(await answer({ repo: "acme/widgets", enabled: true })).toEqual({ status: 200, body: { repo: "acme/widgets", source: "github", codeReview: true } }); - expect(await codeReviewSources(db, TENANT)).toEqual(["github"]); expect((await answer({ repo: "acme/widgets", enabled: false })).body.codeReview).toBe(false); - expect(await codeReviewSources(db, TENANT)).toEqual([]); const tenant = await db.query.tenant.findFirst({ where: eq(schema.tenant.id, TENANT) }); expect(repoRecords(triageNs(tenant?.config)).map((record) => [record.name, record.codeReview])).toEqual([["acme/widgets", false], ["TRI/widgets", undefined], ["acme/gone", undefined]]); expect(changed).toEqual([TENANT, TENANT]); diff --git a/apps/hub/src/code-review.ts b/apps/hub/src/code-review.ts index 64d21f544..27add07fd 100644 --- a/apps/hub/src/code-review.ts +++ b/apps/hub/src/code-review.ts @@ -10,7 +10,7 @@ import { failure } from "./github/portal-credential.js"; import { connectedSources, hasCodeReviewKey } from "./github/source-workflow.js"; import { mailPayload, tenantConfigOf } from "./github/bridge.js"; import { patchCorbitsTriage, repoRecords, sourceOf, triageNs, type RepoRecord } from "./github/tenant-config.js"; -import type { PullRequestHead } from "./code-review-feed.js"; +import type { PullRequestHead } from "./pull-request-reader.js"; import type { PolicyReader } from "./github/check-pack-store.js"; import { NoLiveDeploymentError } from "./github/deployment.js"; import { triageInputs } from "./github/pr-triage.js"; diff --git a/apps/hub/src/github/bridge.test.ts b/apps/hub/src/github/bridge.test.ts index 2fe6930de..710eff84d 100644 --- a/apps/hub/src/github/bridge.test.ts +++ b/apps/hub/src/github/bridge.test.ts @@ -161,7 +161,6 @@ function bridge({ refreshed = [], ...overrides }: Partial & { sent?: cipher: stubCipher, cache: new DeliveryCache(), coalescer: { accept }, - codeReview: { async offer() {} }, log: ignoreLog, readCheckPack: packsFor(["octocat/hello"]), openHeadFor: openHeadOf([]), @@ -469,21 +468,14 @@ describe("bridge handler", () => { expect([sent, rechecks, refreshed]).toEqual([[], [], []]); }); - test("a disabled repo is 202 ignored, and its pull request event still reaches code review", async () => { + test("a disabled repo is 202 ignored", async () => { const sent: Sent[] = []; - const offered: string[] = []; const db = stubDb([hookRow()], { corbitsTriage: { repos: [{ name: "octocat/hello", connected: true, enabled: false }] }, }); - const codeReview = { - async offer(_tenantId: string, record: { name: string }, mail: { action: string | null }) { - offered.push(`${record.name}:${mail.action}`); - }, - }; - const res = await bridge({ db, sent, codeReview })(githubRequest(prPayload), TARGET); + const res = await bridge({ db, sent })(githubRequest(prPayload), TARGET); expect(await res.json()).toEqual({ status: "ignored" }); expect(sent).toHaveLength(0); - expect(offered).toEqual([`octocat/hello:${JSON.parse(prPayload).action}`]); }); test("a draft is 202 ignored as its event says, and triaged once marked ready; a check run is left to its window to read", async () => { diff --git a/apps/hub/src/github/bridge.ts b/apps/hub/src/github/bridge.ts index 593a16664..809455bd7 100644 --- a/apps/hub/src/github/bridge.ts +++ b/apps/hub/src/github/bridge.ts @@ -45,7 +45,6 @@ import type { PolicyReader, StoredPack } from "./check-pack-store.js"; import type { Accepted, Coalescer } from "./coalescer.js"; import type { OpenHeadReader } from "./tenant-open-heads.js"; import type { RepoReads } from "./repo-reads.js"; -import type { CodeReviewFeed } from "../code-review-feed.js"; const HOOK_VERIFY = ["bearer", "standard-webhooks", "slack", "x-hub-signature"] as const; @@ -68,8 +67,6 @@ export interface BridgeDeps { cipher: CredentialCipher; cache: DeliveryCache; coalescer: Pick; - /** Code review is fed whether or not triage is on for the repository. */ - codeReview: Pick; log?: (entry: Record) => void; readCheckPack: PolicyReader; openHeadFor: (tenantId: string) => Promise; @@ -552,7 +549,6 @@ export function createBridgeHandler(d: BridgeDeps) { } const record = await connectedRecord(d, log, loaded, delivery, event, mail.repo); if (record instanceof Response) return record; - void d.codeReview.offer(loaded.tenantId, record, mail); const policy = repoPolicy(record); if (!policy.enabled) { log({ level: "info", msg: "ignored", delivery, event, repo: mail.repo, hook: loaded.credentialId, reason: "repo_not_enabled" }); diff --git a/apps/hub/src/github/coalescer.ts b/apps/hub/src/github/coalescer.ts index 19312573e..76281f048 100644 --- a/apps/hub/src/github/coalescer.ts +++ b/apps/hub/src/github/coalescer.ts @@ -23,7 +23,7 @@ import { dependsOnCi, marksWaiting, type ActionKind, type CheckPack, type PolicySource, type PolicyWorkflow, type PrTriageRow, type RepoPolicy, type TriageEvent } from "@corbits/triage-contracts"; import { summarizeChecks, type CheckRun } from "@corbits/github-reads"; import { mailPayload, unreadablePayload, waitingPayload, type PolicyUnreadable } from "./bridge.js"; -import type { TriageJob } from "./coordinator.js"; +import type { TriageJob } from "./triage-queue.js"; import { GithubRateLimited } from "./github-usage.js"; import type { PrMail } from "./normalize.js"; import { sourceOf, triageStateKey, type RepoRecord } from "./tenant-config.js"; @@ -207,7 +207,7 @@ export function createCoalescer(deps: CoalescerDeps): Coalescer { async function markWaiting({ latest, head }: Open, noCi: boolean): Promise { if (!("pack" in latest) || !marksWaiting(sourceOf(latest.record), latest.automatic)) return; try { - await deps.sendMail({ ...head, workflow: latest.workflow, source: "event", payload: waitingPayload(latest.record, latest.policy, latest, head, noCi) }); + await deps.sendMail({ ...head, payload: waitingPayload(latest.record, latest.policy, latest, head, noCi) }); } catch (err) { deps.log({ level: "warn", msg: "waiting_mail_failed", tenantId: head.tenantId, repo: latest.record.name, pr: head.number, headSha: head.headSha, error: String(err) }); } @@ -262,7 +262,7 @@ export function createCoalescer(deps: CoalescerDeps): Coalescer { const noWait = waitedOut(firstSeenAt, deps.now(), deps.ciWait); const extra = { ...latest.mail, headSha: head.headSha, events, ...(noWait && { noWait }) }; const payload = "pack" in latest ? mailPayload(latest.record, latest.policy, latest, extra) : unreadablePayload(latest.record, latest.policy, latest, extra); - await deps.sendMail({ ...head, workflow: latest.workflow, source: "event", record: latest.record, payload }); + await deps.sendMail({ ...head, record: latest.record, payload }); } catch (err) { if (err instanceof GithubRateLimited) return drop({ ...open, remembered }, entry, "github_rate_limited"); deps.log({ level: "error", msg: "forward_failed", ...entry, error: String(err) }); diff --git a/apps/hub/src/github/coordinator.test.ts b/apps/hub/src/github/coordinator.test.ts deleted file mode 100644 index add423ff7..000000000 --- a/apps/hub/src/github/coordinator.test.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { expect, test } from "bun:test"; -import { createCoordinator, OUTDATED_REASON, TIMED_OUT_REASON, WorkerDeferredError, WorkerOutdatedError, type TriageJob } from "./coordinator.js"; - -const START = new Date("2026-10-14T12:00:00Z").getTime(); -const RUN_TIMEOUT_MS = 60 * 60_000; - -type Timer = { at: number; fire: () => Promise }; - -function fakeClock() { - let now = START; - const timers = new Set(); - function setTimer(fire: () => Promise, ms: number) { - const timer = { at: now + ms, fire }; - timers.add(timer); - return function cancel() { - timers.delete(timer); - }; - } - async function advance(ms: number) { - const until = now + ms; - for (;;) { - const [next] = [...timers].filter((timer) => timer.at <= until).sort((a, b) => a.at - b.at); - if (!next) break; - timers.delete(next); - now = next.at; - await next.fire(); - } - now = until; - } - return { now: () => new Date(now), setTimer, advance }; -} - -function fakeSpawner() { - const spawned: TriageJob[] = []; - const released: string[] = []; - const ends = new Map void; reject: (err: Error) => void }>(); - return { - spawned, - released, - async end(runId: string) { - ends.get(runId)!.resolve(); - await Bun.sleep(0); - }, - async lose(runId: string) { - ends.get(runId)!.reject(new Error("run row gone")); - await Bun.sleep(0); - }, - spawner: { - async spawn(job: TriageJob) { - spawned.push(job); - return `run_${spawned.length}`; - }, - ended(_tenantId: string, runId: string) { - return new Promise(function untilEnded(resolve, reject) { - ends.set(runId, { resolve, reject }); - }); - }, - async release(_tenantId: string, runId: string) { - released.push(runId); - }, - }, - }; -} - -function job(tenantId: string, number: number, payload = {}): TriageJob { - return { tenantId, workflow: "pr-triage-github", repo: "acme/widgets", number, headSha: "sha1", source: "event", payload }; -} - -async function noFailure(): Promise {} - -function setup(failed: (j: TriageJob, error: string) => Promise = noFailure) { - const clock = fakeClock(); - const fake = fakeSpawner(); - const failures: string[] = []; - const coordinator = createCoordinator({ - spawner: fake.spawner, - retry: { minMs: 1_000, maxMs: 3_000 }, - runTimeoutMs: RUN_TIMEOUT_MS, - async failed(j, error) { - failures.push(`${j.number}:${error}`); - await failed(j, error); - }, - setTimer: clock.setTimer, - now: clock.now, - log() {}, - }); - return { clock, fake, failed: failures, coordinator }; -} - -test("a failed spawn fails the head", async () => { - const { clock, fake, failed, coordinator } = setup(); - fake.spawner.spawn = async function noSeed(): Promise { - throw new Error("no seed"); - }; - coordinator.enqueue(job("t1", 1)); - await clock.advance(0); - await Bun.sleep(0); - expect(failed).toEqual(["1:spawn failed: Error: no seed"]); -}); - -test("a job for a pull request whose worker is running waits for it, then spawns", async () => { - const { clock, fake, coordinator } = setup(); - coordinator.enqueue(job("t1", 1, { head: "old" })); - await clock.advance(0); - coordinator.enqueue(job("t1", 1, { head: "new" })); - coordinator.enqueue(job("t1", 2)); - await clock.advance(0); - expect(fake.spawned.map((j) => j.number)).toEqual([1, 2]); - await fake.end("run_1"); - expect(fake.spawned.map((j) => j.number)).toEqual([1, 2, 1]); - expect(fake.spawned.at(-1)).toMatchObject({ number: 1, payload: { head: "new" } }); -}); - -test("a worker whose end could not be observed is still released", async () => { - const { clock, fake, coordinator } = setup(); - coordinator.enqueue(job("t1", 1)); - await clock.advance(0); - await fake.lose("run_1"); - expect(fake.released).toEqual(["run_1"]); -}); - -test("a head that could not be failed does not stall the queue", async () => { - async function stateConflict(): Promise { - throw new Error("state conflict"); - } - const { clock, fake, failed, coordinator } = setup(stateConflict); - const spawn = fake.spawner.spawn; - fake.spawner.spawn = async function noSeedForFirst(j: TriageJob): Promise { - if (j.number === 1) throw new Error("no seed"); - return spawn(j); - }; - coordinator.enqueue(job("t1", 1)); - coordinator.enqueue(job("t1", 2)); - await clock.advance(0); - await Bun.sleep(0); - expect(failed).toEqual(["1:spawn failed: Error: no seed"]); - expect(fake.spawned.map((j) => j.number)).toEqual([2]); -}); - -test("a spawn refused for capacity is retried with backoff, not failed", async () => { - const { clock, fake, failed, coordinator } = setup(); - const spawn = fake.spawner.spawn; - let refusals = 0; - fake.spawner.spawn = async function refuseThrice(j: TriageJob): Promise { - if (refusals < 3) { - refusals += 1; - throw new WorkerDeferredError("no sidecar slots"); - } - return spawn(j); - }; - coordinator.enqueue(job("t1", 1)); - await clock.advance(0); - expect(fake.spawned).toHaveLength(0); - await clock.advance(1_000); - await clock.advance(2_000); - expect(refusals).toBe(3); - expect(fake.spawned).toHaveLength(0); - // The fourth try waits the capped 3 s, not 4 s. - await clock.advance(3_000); - expect(fake.spawned.map((j) => j.number)).toEqual([1]); - expect(failed).toEqual([]); -}); - -test("a worker the sender gate keeps refusing is retried five times, then its head fails and no further worker is provisioned", async () => { - const { clock, fake, failed, coordinator } = setup(); - let refusals = 0; - fake.spawner.spawn = async function outdated(): Promise { - refusals += 1; - throw new WorkerOutdatedError("Workflow run does not admit mail from this sender"); - }; - coordinator.enqueue(job("t1", 1)); - await clock.advance(0); - for (let retry = 0; retry < 10; retry += 1) await clock.advance(3_000); - await Bun.sleep(0); - expect(refusals).toBe(6); - expect(failed).toEqual([`1:${OUTDATED_REASON}`]); -}); - -test("a job from another source for a running pull request waits for its worker", async () => { - const { clock, fake, coordinator } = setup(); - coordinator.enqueue(job("t1", 1)); - await clock.advance(0); - coordinator.enqueue({ ...job("t1", 1, { head: "retry" }), source: "retry" }); - coordinator.enqueue({ ...job("t1", 1, { head: "reconcile" }), source: "reconcile" }); - await clock.advance(0); - expect(fake.spawned.map((j) => j.source)).toEqual(["event"]); - await fake.end("run_1"); - expect(fake.spawned.at(-1)).toMatchObject({ source: "reconcile", payload: { head: "reconcile" } }); - expect(fake.spawned).toHaveLength(2); -}); - -test("a pull request's code review is a job of its own beside its triage", async () => { - const { clock, fake, coordinator } = setup(); - coordinator.enqueue(job("t1", 1)); - coordinator.enqueue({ ...job("t1", 1), workflow: "pr-code-review-github" }); - await clock.advance(0); - expect(fake.spawned.map((j) => j.workflow)).toEqual(["pr-triage-github", "pr-code-review-github"]); -}); - -test("same-named pull requests on two hosts get code reviews that neither replace nor wait for each other", async () => { - const { clock, fake, coordinator } = setup(); - const github = { ...job("t1", 1), workflow: "pr-code-review-github" }; - coordinator.enqueue({ ...github, headSha: "h1" }); - await clock.advance(0); - coordinator.enqueue({ ...github, headSha: "h2" }); - coordinator.enqueue({ ...github, headSha: "h3" }); - coordinator.enqueue({ ...github, workflow: "pr-code-review-bitbucket-datacenter", headSha: "b1" }); - await clock.advance(0); - expect(fake.spawned.map((j) => j.headSha)).toEqual(["h1", "b1"]); - await fake.end("run_1"); - expect(fake.spawned.map((j) => j.headSha)).toEqual(["h1", "b1", "h3"]); -}); - -test("a newer head's job replaces a queued one whatever workflow of the lane each names, so one pull request gets one run", async () => { - const { clock, fake, coordinator } = setup(); - const spawn = fake.spawner.spawn; - fake.spawner.spawn = async function refuseFirst(j: TriageJob): Promise { - if (j.headSha === "sha1") throw new WorkerDeferredError("no sidecar slots"); - return spawn(j); - }; - coordinator.enqueue(job("t1", 1)); - await clock.advance(0); - coordinator.enqueue({ ...job("t1", 1), workflow: "pr-workflow-anthropic-github", headSha: "sha2" }); - await clock.advance(3_000); - expect(fake.spawned.map((j) => [j.workflow, j.headSha])).toEqual([["pr-workflow-anthropic-github", "sha2"]]); -}); - -test("a deferred job whose head a newer job replaced while it spawned is dropped, never run later", async () => { - const { clock, fake, failed, coordinator } = setup(); - const spawn = fake.spawner.spawn; - let refuse: (() => void) | undefined; - fake.spawner.spawn = async function deferFirstLater(j: TriageJob): Promise { - if (j.headSha !== "sha1") return spawn(j); - await new Promise(function untilRefused(resolve) { - refuse = resolve; - }); - throw new WorkerDeferredError("no sidecar slots"); - }; - coordinator.enqueue(job("t1", 1)); - await clock.advance(0); - coordinator.enqueue({ ...job("t1", 1), workflow: "pr-workflow-anthropic-github", headSha: "sha2" }); - refuse!(); - await Bun.sleep(0); - await clock.advance(10_000); - await fake.end("run_1"); - await clock.advance(10_000); - expect(fake.spawned.map((j) => j.headSha)).toEqual(["sha2"]); - expect(failed).toEqual([]); -}); - -test("a worker that never finishes is cancelled at the deadline and the pull request's next job spawns", async () => { - const { clock, fake, failed, coordinator } = setup(); - coordinator.enqueue(job("t1", 1)); - await clock.advance(RUN_TIMEOUT_MS - 1); - expect(fake.released).toEqual([]); - await clock.advance(1); - expect(fake.released).toEqual(["run_1"]); - expect(failed).toEqual([`1:${TIMED_OUT_REASON}`]); - coordinator.enqueue({ ...job("t1", 1), headSha: "sha2" }); - await clock.advance(0); - expect(fake.spawned.map((j) => j.headSha)).toEqual(["sha1", "sha2"]); - await fake.end("run_1"); - expect(fake.released).toEqual(["run_1"]); -}); - -test("a worker that ends before the deadline is neither cancelled early nor failed", async () => { - const { clock, fake, failed, coordinator } = setup(); - coordinator.enqueue(job("t1", 1)); - await clock.advance(RUN_TIMEOUT_MS - 1); - await fake.end("run_1"); - await clock.advance(RUN_TIMEOUT_MS); - expect(fake.released).toEqual(["run_1"]); - expect(failed).toEqual([]); -}); - -test("a spawn that hangs past the deadline frees the slot, and its late worker is released", async () => { - const { clock, fake, failed, coordinator } = setup(); - const spawn = fake.spawner.spawn; - let deliver: (() => void) | undefined; - fake.spawner.spawn = async function hangFirst(j: TriageJob): Promise { - if (j.headSha === "sha1") { - await new Promise(function untilDelivered(resolve) { - deliver = resolve; - }); - } - return spawn(j); - }; - coordinator.enqueue(job("t1", 1)); - await clock.advance(RUN_TIMEOUT_MS); - expect(failed).toEqual([`1:${TIMED_OUT_REASON}`]); - coordinator.enqueue({ ...job("t1", 1), headSha: "sha2" }); - await clock.advance(0); - deliver!(); - await Bun.sleep(0); - expect(fake.spawned.map((j) => j.headSha)).toEqual(["sha2", "sha1"]); - expect(fake.released).toEqual(["run_2"]); - expect(failed).toEqual([`1:${TIMED_OUT_REASON}`]); -}); - -test("a worker whose release fails at the deadline still frees the slot", async () => { - const { clock, fake, failed, coordinator } = setup(); - fake.spawner.release = async function unreachable(): Promise { - throw new Error("hub unreachable"); - }; - coordinator.enqueue(job("t1", 1)); - await clock.advance(RUN_TIMEOUT_MS); - expect(failed).toEqual([`1:${TIMED_OUT_REASON}`]); - coordinator.enqueue({ ...job("t1", 1), headSha: "sha2" }); - await clock.advance(0); - expect(fake.spawned.map((j) => j.headSha)).toEqual(["sha1", "sha2"]); -}); diff --git a/apps/hub/src/github/coordinator.ts b/apps/hub/src/github/coordinator.ts deleted file mode 100644 index 3fe625369..000000000 --- a/apps/hub/src/github/coordinator.ts +++ /dev/null @@ -1,267 +0,0 @@ -// One queue for the hub over the triage jobs the coalescer closes: each job is -// one pull request's folded mail, spawned as a fresh worker deployment and -// released when its one run ends. Capacity is the provisioner's: a spawn it -// refuses for want of room, or one with no listener to copy yet, waits and is -// retried, the latter only so many times before its head fails. A queued job -// for a pull request whose head moved is replaced, not added, as the coalescer -// keys its windows, whichever workflow of its lane either names, and a deferred -// one a newer head replaced is dropped; a job for a pull request whose worker -// is still running waits for it, and the reconciler supersedes the old run's -// result. The queue lives in process: the hub runs as one replica, and the -// heads stay `queued` rows for the reconciler. -import { CODE_REVIEW_WORKFLOWS } from "./source-workflow.js"; -import type { RepoRecord } from "./tenant-config.js"; - -/** Where a job came from; every source goes through the one queue. */ -export type TriageSource = "event" | "retry" | "reconcile"; - -export type TriageJob = { - tenantId: string; - workflow: string; - repo: string; - number: number; - headSha: string; - source: TriageSource; - payload: unknown; - /** The repository a triage job's mail names, which the hub reads the pull request through. */ - record?: RepoRecord; -}; - -export const OUTDATED_REASON = "this repository's Triage deployment is out of date; open the portal to redeploy it"; - -export const NO_LISTENER_REASON = "this repository's Triage deployment is not running; open the portal to redeploy it"; - -/** - * A spawn that cannot happen yet: the host has no room, or its deployment is - * not fit to copy. The job is retried; one waiting on room is never failed, - * one with a `reason` is failed with it after `DEPLOYMENT_ATTEMPTS`. - */ -export class WorkerDeferredError extends Error { - constructor(message: string, readonly reason?: string) { - super(message); - } -} - -/** A worker the sender gate refused: copied from a deployment made before its workflow declared who may mail it, which the portal redeploys. */ -export class WorkerOutdatedError extends WorkerDeferredError { - constructor(message: string) { - super(message, OUTDATED_REASON); - } -} - -/** - * Enough deferrals to cover a portal reload after a hub deploy, and usually a - * failed deployment's heal; then the head fails and the reconciler queues it - * once a deployment is live. One budget per job, whichever reason each - * deferral gives. - */ -const DEPLOYMENT_ATTEMPTS = 5; - -export const TIMED_OUT_REASON = "worker did not finish in time"; - -export type WorkerSpawner = { - /** Deploys a fresh worker and fires it with the job's mail; resolves with the worker's anchor run id. */ - spawn: (job: TriageJob) => Promise; - /** Resolves when the worker's one run has ended. */ - ended: (tenantId: string, runId: string) => Promise; - /** Releases the worker deployment once its run has ended. */ - release: (tenantId: string, runId: string) => Promise; -}; - -export type CoordinatorDeps = { - spawner: WorkerSpawner; - /** How long a deferred job waits before its next spawn: doubled each time from `min`, never past `max`. */ - retry: { minMs: number; maxMs: number }; - /** The job's head is failed here when its worker could not be spawned or fired, or did not finish in time. */ - failed: (job: TriageJob, error: string) => Promise; - /** A worker not finished this long after its spawn began is cancelled; stock `maxLifetime` is per definition, which workers share with the long-lived listener. */ - runTimeoutMs: number; - setTimer: (fire: () => Promise, ms: number) => () => void; - now: () => Date; - log: (entry: Record) => void; -}; - -export type Coordinator = { - enqueue: (job: TriageJob) => void; - /** Cancels the spawn timer; queued heads stay queued for the reconciler. */ - stop: () => void; -}; - -/** One job per pull request for its triage, whichever triage workflow it names, and one for its code review. */ -function keyOf({ tenantId, workflow: id, repo, number }: TriageJob): string { - // A triage job's repo is already unique per host. - const review = CODE_REVIEW_WORKFLOWS.find((workflow) => workflow.id === id); - const lane = review ? `code-review\n${review.source}` : "triage"; - return `${tenantId}\n${lane}\n${repo}\n${number}`; -} - -export function createCoordinator(deps: CoordinatorDeps): Coordinator { - const queue = new Map(); - const running = new Map(); - let slots = 0; - /** Deferred jobs: how often, and when they may spawn again. */ - const refused = new Map(); - /** How often each job was deferred on the host's deployment. */ - const awaitingDeployment = new Map(); - let cancelTimer: (() => void) | undefined; - let pumping = false; - let stopped = false; - - - /** A newer head of the pull request is queued: the job it replaced is dropped, never run. */ - function superseded(job: TriageJob): boolean { - const queued = queue.get(keyOf(job)); - return queued !== undefined && queued.job.headSha !== job.headSha; - } - - function isRunning(job: TriageJob): boolean { - for (const worker of running.values()) if (keyOf(worker.job) === keyOf(job)) return true; - return false; - } - - function next(now: number): { job?: TriageJob; waitMs?: number } { - let waitMs: number | undefined; - for (const { job } of queue.values()) { - if (isRunning(job)) continue; - const notBefore = refused.get(keyOf(job))?.notBefore ?? -Infinity; - if (notBefore <= now) return { job }; - waitMs = Math.min(waitMs ?? Infinity, notBefore - now); - } - return { waitMs }; - } - - function requeue(job: TriageJob): void { - const key = keyOf(job); - if (queue.has(key)) return; - queue.set(key, { job }); - } - - function defer(job: TriageJob): number { - const key = keyOf(job); - const attempts = (refused.get(key)?.attempts ?? 0) + 1; - const waitMs = Math.min(deps.retry.minMs * 2 ** (attempts - 1), deps.retry.maxMs); - refused.set(key, { attempts, notBefore: deps.now().getTime() + waitMs }); - requeue(job); - return waitMs; - } - - async function failJob(job: TriageJob, error: string, entry: Record): Promise { - try { - await deps.failed(job, error); - } catch (err) { - deps.log({ level: "error", msg: "worker_fail_unrecorded", ...entry, error: String(err) }); - } - } - - async function release(job: TriageJob, runId: string, entry: Record): Promise { - try { - await deps.spawner.release(job.tenantId, runId); - } catch (err) { - deps.log({ level: "error", msg: "worker_release_failed", ...entry, runId, error: String(err) }); - } - } - - async function spawn(slot: number, job: TriageJob): Promise { - const entry = { tenantId: job.tenantId, repo: job.repo, pr: job.number, workflow: job.workflow }; - let runId: string | undefined; - let timedOut = false; - const cancelDeadline = deps.setTimer(async function onRunTimeout() { - timedOut = true; - deps.log({ level: "error", msg: "worker_timed_out", ...entry, runId }); - if (runId !== undefined) await release(job, runId, entry); - refused.delete(keyOf(job)); - awaitingDeployment.delete(keyOf(job)); - running.delete(slot); - await failJob(job, TIMED_OUT_REASON, entry); - void pump(); - }, deps.runTimeoutMs); - try { - runId = await deps.spawner.spawn(job); - } catch (err) { - if (timedOut) return; - cancelDeadline(); - running.delete(slot); - const key = keyOf(job); - const deferrals = awaitingDeployment.get(key); - if (err instanceof WorkerDeferredError && superseded(job)) { - refused.delete(key); - awaitingDeployment.delete(key); - deps.log({ level: "info", msg: "worker_job_superseded", ...entry, headSha: job.headSha }); - } else if (err instanceof WorkerDeferredError && err.reason !== undefined && deferrals !== undefined && deferrals >= DEPLOYMENT_ATTEMPTS) { - refused.delete(key); - awaitingDeployment.delete(key); - deps.log({ level: "error", msg: "worker_deployment_unavailable", ...entry, error: String(err) }); - await failJob(job, err.reason, entry); - } else if (err instanceof WorkerDeferredError) { - if (err.reason !== undefined) awaitingDeployment.set(key, deferrals === undefined ? 1 : deferrals + 1); - deps.log({ level: "warn", msg: "worker_spawn_deferred", ...entry, error: String(err), retryMs: defer(job) }); - } else { - deps.log({ level: "error", msg: "worker_spawn_failed", ...entry, error: String(err) }); - await failJob(job, `spawn failed: ${String(err)}`, entry); - } - void pump(); - return; - } - if (timedOut) { - await release(job, runId, entry); - return; - } - refused.delete(keyOf(job)); - awaitingDeployment.delete(keyOf(job)); - deps.log({ level: "info", msg: "worker_spawned", ...entry, runId }); - try { - await deps.spawner.ended(job.tenantId, runId); - } catch (err) { - deps.log({ level: "error", msg: "worker_wait_failed", ...entry, runId, error: String(err) }); - } - if (timedOut) return; - cancelDeadline(); - await release(job, runId, entry); - running.delete(slot); - deps.log({ level: "info", msg: "worker_ended", ...entry, runId }); - void pump(); - } - - function waitThenPump(ms: number): void { - cancelTimer?.(); - cancelTimer = deps.setTimer(async function onSpawnSlot() { - cancelTimer = undefined; - await pump(); - }, ms); - } - - async function pump(): Promise { - if (stopped || pumping) return; - pumping = true; - try { - for (;;) { - const { job, waitMs } = next(deps.now().getTime()); - if (!job) { - if (waitMs !== undefined) waitThenPump(waitMs); - return; - } - queue.delete(keyOf(job)); - // Marks the pull request running through the spawn so a concurrent pump cannot start it twice. - const slot = ++slots; - running.set(slot, { job }); - void spawn(slot, job); - } - } finally { - pumping = false; - } - } - - return { - enqueue(job) { - // A newer job for a waiting pull request takes its place in line, not a new one. - const key = keyOf(job); - queue.set(key, { job }); - void pump(); - }, - stop() { - stopped = true; - cancelTimer?.(); - cancelTimer = undefined; - }, - }; -} diff --git a/apps/hub/src/github/deployment.test.ts b/apps/hub/src/github/deployment.test.ts index a32e8fc91..da6a171bb 100644 --- a/apps/hub/src/github/deployment.test.ts +++ b/apps/hub/src/github/deployment.test.ts @@ -1,6 +1,5 @@ import { describe, expect, test } from "bun:test"; -import type { WorkflowAllocationService, WorkflowLifecycleService } from "@intx/hub-sessions"; -import { createDeploymentRotation, pinOf, resolveLiveDeployments } from "./deployment.js"; +import { pinOf, resolveLiveDeployments, sameSource } from "./deployment.js"; const TENANT_ID = "tnt_acme"; const SOURCE = { kind: "asset", assetId: "ast_triage", package: { format: "source", commitSha: "c0ffee", packageName: "@corbits/pr-triage-workflow" } }; @@ -25,26 +24,6 @@ function closurePinning(...topLevel: { name: string; version: string }[]) { return { schemaVersion: "1", topLevel, entries } as never; } -function tarballRotation(specs: Record) { - const prepared: unknown[] = []; - const rotate = createDeploymentRotation({ - db: { - query: { principal: { findFirst: async () => ({ status: "active" }) } }, - select: () => ({ from: () => ({ innerJoin: () => ({ where: async () => [{ assetId: "ast_pr_triage" }] }) }) }), - } as never, - launchSpecs: { get: async (anchorRunId) => (specs[anchorRunId] ?? null) as never }, - allocation: { - async prepareProvisionedDeployment(args: { anchorRunId: string }) { - prepared.push(args); - return { anchorRunId: args.anchorRunId }; - }, - } as unknown as WorkflowAllocationService, - lifecycle: {} as WorkflowLifecycleService, - mayDeploy: async () => true, - }); - return { rotate, prepared }; -} - function tarballSpec(version: string) { return { ...SPEC, @@ -67,77 +46,21 @@ function selectReturning(rows: unknown[]) { return { select: () => chain } as never; } -function rotation(status: string, mayDeploy = true) { - const prepared: unknown[] = []; - const allocation = { - async prepareProvisionedDeployment(args: { anchorRunId: string }) { - prepared.push(args); - return { anchorRunId: args.anchorRunId, deploymentAddress: `${args.anchorRunId}@acme.test`, allocationId: "alc", status: "pending" }; - }, - } as unknown as WorkflowAllocationService; - const rotate = createDeploymentRotation({ - db: { query: { principal: { findFirst: async () => ({ status }) } } } as never, - launchSpecs: { get: async () => SPEC as never }, - allocation, - lifecycle: {} as WorkflowLifecycleService, - mayDeploy: async () => mayDeploy, - }); - return { rotate, prepared }; -} - -describe("deployment rotation", () => { - test("a copy is deployed from the replaced deployment's launch spec as its active, authorized deployer", async () => { - const { rotate, prepared } = rotation("active"); - const redeployed = await rotate.redeploy(TENANT_ID, "run_old"); - expect(prepared).toEqual([{ - tenantId: TENANT_ID, - anchorRunId: expect.not.stringMatching(/^run_old$/), - deploymentDomain: "acme.test", - source: SOURCE, - entry: "./pr-triage.mjs", - definitionAssetId: "ast_triage", - sessionId: expect.any(String), - sourceAuthorityPrincipalId: "prn_deployer", - sourceOfferingIds: ["off_a", "off_b"], - defaultSourceOfferingId: "off_a", - deployContent: { systemPrompt: "Triage pull requests." }, - }]); - expect(redeployed).toEqual({ status: "deployed", runId: (prepared[0] as { anchorRunId: string }).anchorRunId }); - }); - - test("a deployer no longer active, or without the deploy grant, is not deployed as", async () => { - const inactive = rotation("deactivated"); - expect(await inactive.rotate.redeploy(TENANT_ID, "run_old")).toEqual({ status: "skipped", reason: "deployer_inactive" }); - const denied = rotation("active", false); - expect(await denied.rotate.redeploy(TENANT_ID, "run_old")).toEqual({ status: "skipped", reason: "deployer_not_authorized" }); - expect([...inactive.prepared, ...denied.prepared]).toEqual([]); - }); - +describe("deployments", () => { test("two deployments run the same source only when asset, commit and entry match", async () => { const moved = { ...SPEC, frozenApprovalBundle: { ...SPEC.frozenApprovalBundle, source: { ...SOURCE, package: { ...SOURCE.package, commitSha: "beef" } } } }; const specs: Record = { run_a: SPEC, run_b: SPEC, run_moved: moved }; - const rotate = createDeploymentRotation({ - db: {} as never, - launchSpecs: { get: async (anchorRunId) => specs[anchorRunId] as never ?? null }, - allocation: {} as WorkflowAllocationService, - lifecycle: {} as WorkflowLifecycleService, - mayDeploy: async () => true, - }); - expect(await rotate.sameSource("run_a", "run_b")).toBe(true); - expect(await rotate.sameSource("run_a", "run_moved")).toBe(false); - expect(await rotate.sameSource("run_a", "run_missing")).toBe(false); - }); - - test("a tarball-sourced copy carries the replaced deployment's pin and definition asset", async () => { - const { rotate, prepared } = tarballRotation({ run_old: tarballSpec("0.1.0-a") }); - await rotate.redeploy(TENANT_ID, "run_old"); - expect(prepared).toEqual([expect.objectContaining({ source: TARBALL_SOURCE, pin: "@corbits/pr-triage-workflow@0.1.0-a", definitionAssetId: "ast_pr_triage" })]); + const launchSpecs = { get: async (anchorRunId: string) => (specs[anchorRunId] ?? null) as never }; + expect(await sameSource(launchSpecs, "run_a", "run_b")).toBe(true); + expect(await sameSource(launchSpecs, "run_a", "run_moved")).toBe(false); + expect(await sameSource(launchSpecs, "run_a", "run_missing")).toBe(false); }); test("tarball-sourced deployments with different pins are not the same source", async () => { - const { rotate } = tarballRotation({ run_a: tarballSpec("0.1.0-a"), run_b: tarballSpec("0.1.0-a"), run_bumped: tarballSpec("0.1.0-b") }); - expect(await rotate.sameSource("run_a", "run_b")).toBe(true); - expect(await rotate.sameSource("run_a", "run_bumped")).toBe(false); + const specs: Record = { run_a: tarballSpec("0.1.0-a"), run_b: tarballSpec("0.1.0-a"), run_bumped: tarballSpec("0.1.0-b") }; + const launchSpecs = { get: async (anchorRunId: string) => (specs[anchorRunId] ?? null) as never }; + expect(await sameSource(launchSpecs, "run_a", "run_b")).toBe(true); + expect(await sameSource(launchSpecs, "run_a", "run_bumped")).toBe(false); }); test("a deployment whose cancellation was requested is listed as cancelling", async () => { diff --git a/apps/hub/src/github/deployment.ts b/apps/hub/src/github/deployment.ts index 0e59f898f..bc07dee92 100644 --- a/apps/hub/src/github/deployment.ts +++ b/apps/hub/src/github/deployment.ts @@ -1,12 +1,10 @@ import { and, desc, eq, gt, inArray, isNotNull, isNull, ne, or, sql } from "drizzle-orm"; import { schema, type DB, type ResolvedSenderPrincipal, type WorkflowRunLaunchSpecStore } from "@intx/db"; -import { generateId } from "@intx/hub-common"; -import { MAIL_ADMISSION_DENIED_CODE, type WorkflowAllocationService, type WorkflowLifecycleService } from "@intx/hub-sessions"; +import { MAIL_ADMISSION_DENIED_CODE } from "@intx/hub-sessions"; import type { RunTriggerMaterialize } from "@corbits/webhooks"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; import type { FrozenApprovalBundle } from "@intx/types/sidecar"; import { TRIAGE_LISTENERS, triageStopped, type RepoSource } from "@corbits/triage-contracts"; -import { claimRotation, clearRotation, type RotationRecord } from "./tenant-config.js"; export class NoLiveDeploymentError extends Error { constructor(readonly workflow: string) { @@ -63,10 +61,9 @@ export type TenantDeployment = LiveDeployment & { workflow: string }; /** * A worker deployment carries this key in its deploy content, so the hub can tell it from the listener it was copied - * from: a worker is spawned for one mail and released when that run ends, and nothing else may mail it or count it as a - * rotation. + * from: a worker ran one mail and was released when that run ended. */ -export const WORKER_MARKER = "corbitsTriageWorker"; +const WORKER_MARKER = "corbitsTriageWorker"; export type DeploymentKind = "listener" | "worker"; @@ -124,21 +121,11 @@ export function resolveLiveDeployments(db: DB["db"], tenantId: string, definitio return deploymentsWhere(db, { tenantId, definitionName, kind }); } -/** Workers live or released since `endedSince`: a worker's one body run lands in its log before the coordinator cancels the anchor, so the log outlives the deployment. */ +/** Workers live or released since `endedSince`: a worker's one body run landed in its log before its anchor was cancelled, so the log outlives the deployment. */ export function recentWorkers(db: DB["db"], tenantId: string, definitionName: string, endedSince: Date): Promise { return deploymentsWhere(db, { tenantId, definitionName, kind: "worker", endedSince }); } -/** Every tenant's live worker deployments, for the release of workers a hub restart orphaned. */ -export async function liveWorkers(db: DB["db"]): Promise<{ tenantId: string; runId: string }[]> { - const { workflowRun, workflowRunLaunchSpec, liveWorkflowRunStatuses } = schema; - return db - .select({ tenantId: workflowRun.tenantId, runId: workflowRun.id }) - .from(workflowRun) - .innerJoin(workflowRunLaunchSpec, eq(workflowRunLaunchSpec.anchorRunId, workflowRun.id)) - .where(and(eq(workflowRun.id, workflowRun.anchorRunId), inArray(workflowRun.status, [...liveWorkflowRunStatuses]), markedAs("worker"))); -} - export type ActiveTenant = { id: string; domain: string }; /** @@ -228,101 +215,17 @@ export async function withWorkflowVersions( return versioned; } -export type DeploymentRotationDeps = { - db: DB["db"]; - launchSpecs: Pick; - allocation: WorkflowAllocationService; - lifecycle: WorkflowLifecycleService; - /** As the stock deploy route requires. */ - mayDeploy: (principalId: string, tenantId: string) => Promise; - /** Where the tenant config records this deployment's rotation; the GitHub one keeps the original key. */ - rotationKey?: string; -}; - +/** The reason the hub gave when it replaced a busy deployment with a fresh copy, which earlier audit entries carry. */ export const ROTATION_REASON = "Replaced by a fresh deployment of the same workflow"; -export type Redeployed = { status: "deployed"; runId: string } | { status: "skipped"; reason: string }; - -/** - * A deployment's run log grows with every run and the stock hub validates each - * received commit against all of it, so a busy deployment is replaced by a - * fresh one. The copy is what the stock deploy route would create from the same - * request, read from the deployment's launch spec: its source and commit, - * entry, offerings, deploy content and the principal that deployed it. The - * copy keeps that principal because the workflow runs as it; it is used only - * while the stock route would still accept it: an active member holding the - * route's grant. `content` is added to the copy's deploy content; a worker is - * marked that way. - */ -export function createDeploymentRotation(deps: DeploymentRotationDeps) { - /** A tarball source is the shared package registry, so the workflow asset is read from the run's definition. */ - async function definitionAssetId(tenantId: string, anchorRunId: string): Promise { - const { workflowRun, workflowDefinition } = schema; - const [row] = await deps.db - .select({ assetId: workflowDefinition.assetId }) - .from(workflowRun) - .innerJoin(workflowDefinition, eq(workflowRun.definitionId, workflowDefinition.id)) - .where(and(eq(workflowRun.id, anchorRunId), eq(workflowRun.tenantId, tenantId))); - if (!row?.assetId) throw new Error(`deployment ${anchorRunId} has no workflow definition asset`); - return row.assetId; - } - - async function redeploy(tenantId: string, anchorRunId: string, content: Record = {}): Promise { - const spec = await deps.launchSpecs.get(anchorRunId); - if (!spec) throw new Error(`deployment ${anchorRunId} has no launch spec`); - const { source, entry } = spec.frozenApprovalBundle; - if (source.kind !== "asset") throw new Error(`deployment ${anchorRunId} is not deployed from a workflow asset`); - const pin = pinFor(spec.frozenApprovalBundle); - const systemPrompt = spec.deployContent["systemPrompt"]; - if (typeof systemPrompt !== "string") throw new Error(`deployment ${anchorRunId} has no system prompt in its deploy content`); - const deployer = await deps.db.query.principal.findFirst({ - where: and(eq(schema.principal.id, spec.sourceAuthorityPrincipalId), eq(schema.principal.tenantId, tenantId)), - columns: { status: true }, - }); - if (deployer?.status !== "active") return { status: "skipped", reason: "deployer_inactive" }; - if (!await deps.mayDeploy(spec.sourceAuthorityPrincipalId, tenantId)) { - return { status: "skipped", reason: "deployer_not_authorized" }; - } - const definition = pin === undefined ? source.assetId : await definitionAssetId(tenantId, anchorRunId); - const prepared = await deps.allocation.prepareProvisionedDeployment({ - tenantId, - anchorRunId: generateId("workflowRun"), - deploymentDomain: spec.deploymentDomain, - source, - entry, - ...(pin !== undefined ? { pin } : {}), - definitionAssetId: definition, - sessionId: generateId("session"), - sourceAuthorityPrincipalId: spec.sourceAuthorityPrincipalId, - sourceOfferingIds: spec.sourceOfferingIds, - defaultSourceOfferingId: spec.defaultSourceOfferingId, - deployContent: { ...spec.deployContent, ...content, systemPrompt }, - }); - return { status: "deployed", runId: prepared.anchorRunId }; - } - - async function sameSource(anchorRunId: string, otherAnchorRunId: string): Promise { - const [spec, other] = await Promise.all([deps.launchSpecs.get(anchorRunId), deps.launchSpecs.get(otherAnchorRunId)]); - if (!spec || !other) return false; - const a = spec.frozenApprovalBundle; - const b = other.frozenApprovalBundle; - if (a.source.kind !== "asset" || b.source.kind !== "asset" || a.entry !== b.entry) return false; - return a.source.assetId === b.source.assetId - && JSON.stringify(a.source.package) === JSON.stringify(b.source.package) - && pinFor(a) === pinFor(b); - } - - async function release(tenantId: string, anchorRunId: string): Promise { - await deps.lifecycle.requestCancellation(tenantId, anchorRunId, ROTATION_REASON); - } - - function claim(tenantId: string, record: RotationRecord): Promise { - return claimRotation(deps.db, tenantId, record, deps.rotationKey); - } - - function clear(tenantId: string, record: RotationRecord): Promise { - return clearRotation(deps.db, tenantId, record, deps.rotationKey); - } - - return { redeploy, sameSource, release, claim, clear }; +/** Whether two deployments were made from the same request: one source, package, pin and entry. */ +export async function sameSource(launchSpecs: Pick, anchorRunId: string, otherAnchorRunId: string): Promise { + const [spec, other] = await Promise.all([launchSpecs.get(anchorRunId), launchSpecs.get(otherAnchorRunId)]); + if (!spec || !other) return false; + const a = spec.frozenApprovalBundle; + const b = other.frozenApprovalBundle; + if (a.source.kind !== "asset" || b.source.kind !== "asset" || a.entry !== b.entry) return false; + return a.source.assetId === b.source.assetId + && JSON.stringify(a.source.package) === JSON.stringify(b.source.package) + && pinFor(a) === pinFor(b); } diff --git a/apps/hub/src/github/installation-token.postgres.test.ts b/apps/hub/src/github/installation-token.postgres.test.ts index 0c71bfa9c..914a3988f 100644 --- a/apps/hub/src/github/installation-token.postgres.test.ts +++ b/apps/hub/src/github/installation-token.postgres.test.ts @@ -141,10 +141,6 @@ test("PostgreSQL: the App key moves to the hub on boot, each renewed token reach expect(await renewal.renewAll()).toEqual({ retry: false }); expect(pushed().at(-1)).toEqual({ address: LISTENER, secrets: ["ghs_1_7_2"] }); - // A worker is deployed with a token that has most of its hour left as it is. - await renewal.ensure(TENANT); - expect(github.minted()).toBe(2); - // A stock credential write replaces the App while a renewal mints from the old key: the renewal does not overwrite it. github.hooks.onMint = async function stockWriteSavesReplacement() { await db.update(schema.credential).set({ secret: await cipher.encrypt(replacement.json, credentialAad("crd_github", "secret")) }).where(eq(schema.credential.id, "crd_github")); @@ -154,8 +150,8 @@ test("PostgreSQL: the App key moves to the hub on boot, each renewed token reach expect(await secretOf("github")).toBe(replacement.json); expect(pushed()).toHaveLength(3); - // The next worker moves the replacement off the bound credential and mints from it. - await renewal.ensure(TENANT); + // The next renewal moves the replacement off the bound credential and mints from it. + await renewal.renew(TENANT); expect(await secretOf("github-app")).toBe(replacement.json); expect(await secretOf("github")).toBe("ghs_2_7_4"); expect(pushed().at(-1)).toEqual({ address: LISTENER, secrets: ["ghs_2_7_4"] }); @@ -172,12 +168,12 @@ test("PostgreSQL: the App key moves to the hub on boot, each renewed token reach expect(await renewal.renewAll()).toEqual({ retry: true }); expect(await githubRow()).toMatchObject({ status: "active", metadata: { appSlug: "app-2", tokenError: expect.stringContaining("503") } }); await db.update(schema.credential).set({ expiresAt: new Date(Date.now() - 1000) }).where(eq(schema.credential.id, "crd_github")); - await expect(renewal.ensure(TENANT)).rejects.toThrow("503"); + await expect(renewal.renew(TENANT)).rejects.toThrow("503"); expect((await githubRow()).status).toBe("expired"); expect((await launchable()).ok).toBe(false); github.hooks.down = false; - await renewal.ensure(TENANT); + await renewal.renew(TENANT); expect(await githubRow()).toMatchObject({ status: "active", metadata: { appSlug: "app-2" } }); expect((await githubRow()).metadata).not.toHaveProperty("tokenError"); expect((await launchable()).ok).toBe(true); diff --git a/apps/hub/src/github/installation-token.ts b/apps/hub/src/github/installation-token.ts index b53257654..ef3f60264 100644 --- a/apps/hub/src/github/installation-token.ts +++ b/apps/hub/src/github/installation-token.ts @@ -18,8 +18,6 @@ import { repoRecords, sourceOf, triageNs } from "./tenant-config.js"; /** Installation tokens last an hour. */ export const GITHUB_TOKEN_RENEW_MS = 45 * 60_000; -/** A worker is deployed with the token it reads, so one with less left than this is renewed first. */ -const MIN_TOKEN_LIFE_MS = 15 * 60_000; /** A full pass falls due a little before the renewal interval fires, so the interval's pass is always full. */ const RETRY_SLACK_MS = 60_000; @@ -154,22 +152,6 @@ export function createGithubTokenRenewal(deps: GithubTokenDeps) { }); } - /** - * Renews the token of the workspace whose credential `tenantId` resolves to unless it is active with long enough left; - * reading the key first moves one a stock write left, which leaves no token. - */ - async function ensure(tenantId: string): Promise { - const owner = (await heldCredential(deps.db, tenantId, GITHUB_CREDENTIAL))?.tenantId; - if (owner === undefined) return; - await recorded(owner, async function ensureToken() { - await secure(owner); - const appJson = await githubAppKey(keyDeps, owner); - const github = await heldCredential(deps.db, owner, GITHUB_CREDENTIAL); - if (github?.status === "active" && github.expiresAt && github.expiresAt.getTime() - Date.now() > MIN_TOKEN_LIFE_MS) return; - await renewWith(owner, appJson); - }); - } - /** A key whose `github` credential was removed or revoked through the stock route goes with it. */ async function dropOrphanKey(tenantId: string): Promise { await deps.db.transaction(async function drop(tx) { @@ -211,7 +193,7 @@ export function createGithubTokenRenewal(deps: GithubTokenDeps) { return { retry: failed.size > 0, ...(retryAt !== undefined && { retryAt }) }; } - return { issue, push, secure, renew, ensure, renewAll }; + return { issue, push, secure, renew, renewAll }; } export type GithubTokenRenewal = ReturnType; diff --git a/apps/hub/src/github/pr-triage.ts b/apps/hub/src/github/pr-triage.ts index c21a8edb1..762ebff5a 100644 --- a/apps/hub/src/github/pr-triage.ts +++ b/apps/hub/src/github/pr-triage.ts @@ -8,14 +8,13 @@ import { schema } from "@intx/db"; import { hasEnabledRules, repoPolicy, triageStopped, type PrTriageRow, type RepoSource } from "@corbits/triage-contracts"; import { errorText } from "@corbits/common"; import { mailPayload } from "./bridge.js"; -import type { TriageJob } from "./coordinator.js"; +import type { TriageJob } from "./triage-queue.js"; import type { PolicyReader } from "./check-pack-store.js"; import { githubLimited } from "./github-refusal.js"; import { failure, sourceCredentialGrant, type PortalCredentialDeps } from "./portal-credential.js"; import type { Member } from "../integrations/app.js"; import type { ManagerRuns } from "./manager-runs.js"; import { isStuck, runKey, type ObservedRun, type ReconcilePolicy } from "./reconcile-plan.js"; -import { triageWorkflowFor } from "./source-workflow.js"; import { repoRecords, sourceOf, triageNs, triageStateKey, type RepoRecord } from "./tenant-config.js"; import type { LiveDeployment } from "./deployment.js"; import type { ObservedRuns } from "./triage-runs.js"; @@ -92,7 +91,6 @@ export function createGithubPrTriage(deps: GithubPrTriageDeps) { const ready = await readyToTriage(deps, member, repo); if (ready instanceof Response) return ready; const { record, policy, pack } = ready; - const triage = triageWorkflowFor(sourceOf(record)); if (!(await deps.listener(tenantId, sourceOf(record)))) return failure(503, "service_not_running", triageStopped(sourceOf(record))); let openHead: string | undefined; @@ -121,7 +119,7 @@ export function createGithubPrTriage(deps: GithubPrTriageDeps) { } if (queued.status === "conflict") return failure(409, "conflict", `${repo}#${number} was just updated by the hub. Try again.`); try { - await deps.send({ ...head, workflow: triage.id, source: "retry", record, payload: mailPayload(record, policy, pack, { prNumber: number, headSha, noWait: true }) }); + await deps.send({ ...head, record, payload: mailPayload(record, policy, pack, { prNumber: number, headSha, noWait: true }) }); } catch (err) { deps.log({ level: "error", msg: "triage_request_failed", tenantId, repo, pr: number, headSha, error: String(err) }); try { diff --git a/apps/hub/src/github/pr-workflow.test.ts b/apps/hub/src/github/pr-workflow.test.ts index 18dc39cbe..043b074a1 100644 --- a/apps/hub/src/github/pr-workflow.test.ts +++ b/apps/hub/src/github/pr-workflow.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "bun:test"; import type { WorkflowRunEvent } from "@intx/hub-sessions"; import { decideOf, defaultActions, WAITING_KIND, type PolicyWorkflow } from "@corbits/triage-contracts"; -import type { PullRequestToReview } from "../code-review-feed.js"; +import type { PullRequestToReview } from "../pull-request-reader.js"; import { createPauseGate, prWorkflowKeys, workflowPart } from "./pr-workflow.js"; import type { RepoRecord } from "./tenant-config.js"; import { verdictFor } from "./triage-runs.js"; diff --git a/apps/hub/src/github/pr-workflow.ts b/apps/hub/src/github/pr-workflow.ts index ccc758e67..d646521b1 100644 --- a/apps/hub/src/github/pr-workflow.ts +++ b/apps/hub/src/github/pr-workflow.ts @@ -2,7 +2,7 @@ // the listener: a paused pull request starts no run, and in doubt nothing approves or merges unattended. import { isDeepStrictEqual } from "node:util"; import { BLOCK_KINDS, decideOf, EVERYONE, repoSourceOf, VERDICTS, type BlockKind, type CheckPack, type CodeReviewProvider, type PolicyWorkflow, type RepoSource } from "@corbits/triage-contracts"; -import type { PullRequestHead } from "../code-review-feed.js"; +import type { PullRequestHead } from "../pull-request-reader.js"; import type { RepoRecord } from "./tenant-config.js"; /** `blocks` are those the variant runs; a policy turning any other on runs nowhere yet. */ diff --git a/apps/hub/src/github/rotation-record.postgres.test.ts b/apps/hub/src/github/rotation-record.postgres.test.ts deleted file mode 100644 index e5cbaa6dd..000000000 --- a/apps/hub/src/github/rotation-record.postgres.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -// Requires PostgreSQL at TEST_DB. -import { expect, test } from "bun:test"; -import { randomBytes } from "node:crypto"; -import { sql, eq } from "drizzle-orm"; -import { createDB, runMigrations, schema } from "@intx/db"; -import { claimRotation, clearRotation, prepareCorbitsTriagePatch, rotationRecord, triageNs } from "./tenant-config.js"; - -const TEST_DB = { host: "localhost", port: 5432, user: "postgres", password: "postgres", database: "interchange" }; -const T = "tnt_rot"; - -async function noGrant(): Promise {} - -async function allowAll(): Promise { - return true; -} - -function patchReq(ns: Record) { - return new Request("https://hub/api/tenants/x", { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify({ config: { corbitsTriage: ns } }) }); -} - -test("PostgreSQL: a rotation is claimed once, a portal save read before it is refused and one read after keeps it, and only the matching record is cleared", async () => { - const database = `corbits_rot_${randomBytes(6).toString("hex")}`; - const admin = createDB(TEST_DB); - await admin.db.execute(sql.raw(`create database ${database}`)); - const config = { ...TEST_DB, database }; - let a: ReturnType | undefined; - try { - await runMigrations(config, { schema: "public" }); - a = createDB(config); - await a.db.insert(schema.tenant).values({ id: T, name: "R", slug: "r", domain: "r.test", config: { corbitsTriage: { repos: [], rev: 3 } } }); - const stale = { repos: [], rev: 3 }; - const record = { from: "run_a", to: "run_c", at: "2026-10-08T00:00:00.000Z" }; - expect(await claimRotation(a.db, T, record)).toBe(true); - expect(await claimRotation(a.db, T, { ...record, to: "run_d" })).toBe(false); - const refused = await prepareCorbitsTriagePatch({ db: a.db, authorize: allowAll, grantEnabled: noGrant }, T, "prn_admin", patchReq(stale)); - expect(refused instanceof Response && refused.status).toBe(409); - const [row] = await a.db.select().from(schema.tenant).where(eq(schema.tenant.id, T)); - const fresh = triageNs(row!.config); - const passed = await prepareCorbitsTriagePatch({ db: a.db, authorize: allowAll, grantEnabled: noGrant }, T, "prn_admin", patchReq(fresh)); - expect(passed instanceof Response).toBe(false); - await clearRotation(a.db, T, { ...record, to: "other" }); - const [kept] = await a.db.select().from(schema.tenant).where(eq(schema.tenant.id, T)); - expect(rotationRecord(triageNs(kept!.config))).toEqual(record); - await clearRotation(a.db, T, record); - const [cleared] = await a.db.select().from(schema.tenant).where(eq(schema.tenant.id, T)); - expect(rotationRecord(triageNs(cleared!.config))).toBeUndefined(); - // Only the claim, the portal save and the matching clear wrote; the refused claim and the other clear were no-ops. - expect(triageNs(cleared!.config).rev).toBe(6); - } finally { - await a?.close(); - await admin.db.execute(sql.raw(`drop database if exists ${database} with (force)`)); - await admin.close(); - } -}, 120_000); diff --git a/apps/hub/src/github/source-workflow.test.ts b/apps/hub/src/github/source-workflow.test.ts index b063c324d..0283ade5f 100644 --- a/apps/hub/src/github/source-workflow.test.ts +++ b/apps/hub/src/github/source-workflow.test.ts @@ -1,54 +1,32 @@ import { expect, test } from "bun:test"; -import { createSourceDeployments, triageWorkflowOf } from "./source-workflow.js"; - -type Live = Record>; - -type Host = "github" | "bitbucket-datacenter"; - -function harness(connected: Host[], live: Live, reviewed: Host[] = [], placed = true) { - const released: string[] = []; - const logged: Array> = []; - const reconcile = createSourceDeployments({ - connectedSources: async () => connected, - codeReviewSources: async () => reviewed, - codeReviewPlaced: async () => placed, - liveDeployments: async (_tenantId, workflow) => live[workflow] ?? [], - async release(_tenantId, runId) { - released.push(runId); - }, - log(entry) { - logged.push(entry); - }, - }); - return { reconcile, released, logged }; -} - -test("disconnecting a host releases its deployment; the pre-host pr-triage deployment goes once pr-triage-github is live", async () => { - const h = harness(["github"], { "pr-triage-github": [{ runId: "run_gh", cancelling: false }], "pr-triage-bitbucket-datacenter": [{ runId: "run_bb", cancelling: false }], "pr-triage": [{ runId: "run_legacy", cancelling: false }] }); - await h.reconcile("tnt"); - expect(h.released).toEqual(["run_bb", "run_legacy"]); - expect(h.logged.map((entry) => [entry.msg, entry.reason])).toEqual([["triage_deployment_released", "source_disconnected"], ["triage_deployment_released", "superseded_by_pr_triage_github"]]); -}); - -test("a host's code review stays deployed only while one of its repositories has it on and a provisioner can run it", async () => { - const live = { "pr-triage-github": [{ runId: "run_gh", cancelling: false }], "pr-code-review-github": [{ runId: "run_review", cancelling: false }] }; - const on = harness(["github"], live, ["github"]); - await on.reconcile("tnt"); - expect(on.released).toEqual([]); - - const off = harness(["github"], live); - await off.reconcile("tnt"); - expect(off.released).toEqual(["run_review"]); - expect(off.logged.map((entry) => entry.reason)).toEqual(["code_review_off"]); - - const nowhere = harness(["github"], live, ["github"], false); - await nowhere.reconcile("tnt"); - expect(nowhere.released).toEqual(["run_review"]); - expect(nowhere.logged.map((entry) => entry.reason)).toEqual(["no_isolated_place"]); -}); +import { createHostRelease, triageWorkflowOf } from "./source-workflow.js"; test("the pre-host workflow name and each package entry name a host's workflow", () => { expect(triageWorkflowOf("pr-triage")?.id).toBe("pr-triage-github"); expect(triageWorkflowOf("./pr-triage-bitbucket-datacenter.mjs")?.source).toBe("bitbucket-datacenter"); expect(triageWorkflowOf("./pr-triage-historical.mjs")).toBeUndefined(); }); + +test("disconnecting a host releases its listener and managers and leaves the other host's running", async () => { + const live: Record> = { + "triage-listener-github": [{ runId: "run_gh_listener", cancelling: false }], + "triage-manager-github": [{ runId: "run_gh_manager", cancelling: false }], + "triage-listener-bitbucket-datacenter": [{ runId: "run_bb_listener", cancelling: false }], + "triage-manager-bitbucket-datacenter": [{ runId: "run_bb_manager", cancelling: false }, { runId: "run_bb_stopping", cancelling: true }], + }; + const cancelled: string[] = []; + const release = createHostRelease({ + async connectedSources() { + return ["github"]; + }, + async liveDeployments(_tenantId, workflow) { + return live[workflow]!; + }, + async cancel(_tenantId, runId) { + cancelled.push(runId); + }, + log() {}, + }); + await release("tnt"); + expect(cancelled).toEqual(["run_bb_listener", "run_bb_manager"]); +}); diff --git a/apps/hub/src/github/source-workflow.ts b/apps/hub/src/github/source-workflow.ts index f2954460b..a78d6c560 100644 --- a/apps/hub/src/github/source-workflow.ts +++ b/apps/hub/src/github/source-workflow.ts @@ -3,13 +3,12 @@ // its host's workflow. import { and, eq, inArray } from "drizzle-orm"; import { schema, type DB } from "@intx/db"; -import { CODE_REVIEW_PROVIDERS, codeReviewProviderOf, TRIAGE_MANAGERS, type RepoSource } from "@corbits/triage-contracts"; +import { CODE_REVIEW_PROVIDERS, codeReviewProviderOf, REPO_SOURCES, TRIAGE_LISTENERS, TRIAGE_MANAGERS, type RepoSource } from "@corbits/triage-contracts"; import { PR_TRIAGE_ADDRESS } from "@corbits/triage-workflows/pr-triage"; import { PR_TRIAGE_BITBUCKET_DATACENTER_ADDRESS, workflow as prTriageBitbucket } from "@corbits/triage-workflows/pr-triage-bitbucket-datacenter"; import { PR_TRIAGE_GITHUB_ADDRESS, workflow as prTriageGithub } from "@corbits/triage-workflows/pr-triage-github"; import { PR_CODE_REVIEW_BITBUCKET_DATACENTER_ADDRESS, PR_CODE_REVIEW_GITHUB_ADDRESS } from "@corbits/pr-code-review/pr-code-review"; import { PR_WORKFLOW_VARIANTS, prWorkflowVariantOf } from "./pr-workflow.js"; -import { repoRecords, ROTATION_KEY, sourceOf, triageNs } from "./tenant-config.js"; export type TriageWorkflow = { source: RepoSource; @@ -18,13 +17,11 @@ export type TriageWorkflow = { /** The portal's package entry, which the deploy route sees. */ entry: string; credential: string; - /** Where the tenant config records this workflow's deployment rotation; GitHub's keeps the key from before hosts. */ - rotationKey: string; }; export const TRIAGE_WORKFLOWS: readonly TriageWorkflow[] = [ - { source: "github", id: prTriageGithub.id, address: PR_TRIAGE_GITHUB_ADDRESS, entry: "./pr-triage-github.mjs", credential: "github", rotationKey: ROTATION_KEY }, - { source: "bitbucket-datacenter", id: prTriageBitbucket.id, address: PR_TRIAGE_BITBUCKET_DATACENTER_ADDRESS, entry: "./pr-triage-bitbucket-datacenter.mjs", credential: "bitbucket-datacenter", rotationKey: "rotationBitbucketDatacenter" }, + { source: "github", id: prTriageGithub.id, address: PR_TRIAGE_GITHUB_ADDRESS, entry: "./pr-triage-github.mjs", credential: "github" }, + { source: "bitbucket-datacenter", id: prTriageBitbucket.id, address: PR_TRIAGE_BITBUCKET_DATACENTER_ADDRESS, entry: "./pr-triage-bitbucket-datacenter.mjs", credential: "bitbucket-datacenter" }, ]; /** Code review per host, binding that host's read credential; it is deployed while a repository of the host has it on, from the package of the workspace's code review provider. */ @@ -56,7 +53,7 @@ export function codeReviewWorkflowFor(source: RepoSource): CodeReviewWorkflow { } /** Every workflow a host's pull request job can run in: its triage, its code review and its PR workflow variants. */ -export function jobWorkflowsFor(source: RepoSource): string[] { +function jobWorkflowsFor(source: RepoSource): string[] { const variants = PR_WORKFLOW_VARIANTS.filter((variant) => variant.source === source).map((variant) => variant.id); return [triageWorkflowFor(source).id, codeReviewWorkflowFor(source).id, ...variants]; } @@ -87,14 +84,6 @@ export async function hasCodeReviewKey(db: DB["db"], tenantId: string): Promise< return codeReviewProviderOf(await codeReviewKeys(db, tenantId)) !== undefined; } -/** The hosts with a connected repository that has code review on, while a code review provider's key is stored. */ -export async function codeReviewSources(db: DB["db"], tenantId: string): Promise { - if (!(await hasCodeReviewKey(db, tenantId))) return []; - const tenant = await db.query.tenant.findFirst({ where: eq(schema.tenant.id, tenantId), columns: { config: true } }); - const reviewed = repoRecords(triageNs(tenant?.config)).filter((record) => record.connected && record.codeReview === true); - return [...new Set(reviewed.map(sourceOf))]; -} - /** The hosts whose credential the workspace holds. */ export async function connectedSources(db: DB["db"], tenantId: string): Promise { const credentials = TRIAGE_WORKFLOWS.map((workflow) => workflow.credential); @@ -106,54 +95,25 @@ export async function connectedSources(db: DB["db"], tenantId: string): Promise< return TRIAGE_WORKFLOWS.filter((workflow) => names.has(workflow.credential)).map((workflow) => workflow.source); } -type SourceDeploymentsDeps = { +export type HostReleaseDeps = { connectedSources: (tenantId: string) => Promise; - codeReviewSources: (tenantId: string) => Promise; - codeReviewPlaced: (tenantId: string) => Promise; liveDeployments: (tenantId: string, workflow: string) => Promise>; - release: (tenantId: string, anchorRunId: string) => Promise; + cancel: (tenantId: string, runId: string) => Promise; log: (entry: Record) => void; }; -/** - * Keeps a workspace's pr-triage deployments to the hosts it connected: a disconnected host's live deployment is released, - * and a connected host without one is logged for the portal to deploy, which only it can, as it publishes the packages. - * A deployment of the workflow from before hosts is released once the GitHub one is live. A host's code review is - * released once no repository of it has code review on, the Anthropic key is gone, or no provisioner can run it; a host's - * PR workflow variant once the host is disconnected or no provisioner can run it; the portal releases it with its key. - */ -export function createSourceDeployments(deps: SourceDeploymentsDeps) { - async function releaseAll(tenantId: string, workflow: string, reason: string): Promise { - for (const live of await deps.liveDeployments(tenantId, workflow)) { - if (live.cancelling) continue; - await deps.release(tenantId, live.runId); - deps.log({ level: "info", msg: "triage_deployment_released", tenantId, workflow, deployment: live.runId, reason }); - } - } - - return async function reconcileSourceDeployments(tenantId: string): Promise { +/** A host's listener and the managers it deployed hold its credential, so they are released once the host is disconnected. */ +export function createHostRelease(deps: HostReleaseDeps) { + return async function releaseDisconnected(tenantId: string): Promise { const connected = await deps.connectedSources(tenantId); - for (const workflow of TRIAGE_WORKFLOWS) { - if (!connected.includes(workflow.source)) { - await releaseAll(tenantId, workflow.id, "source_disconnected"); - continue; + for (const source of REPO_SOURCES.filter((host) => !connected.includes(host))) { + for (const workflow of [TRIAGE_LISTENERS[source], TRIAGE_MANAGERS[source]]) { + for (const live of await deps.liveDeployments(tenantId, workflow)) { + if (live.cancelling) continue; + await deps.cancel(tenantId, live.runId); + deps.log({ level: "info", msg: "triage_deployment_released", tenantId, workflow, deployment: live.runId, reason: "source_disconnected" }); + } } - const live = await deps.liveDeployments(tenantId, workflow.id); - if (live.some((deployment) => !deployment.cancelling)) continue; - deps.log({ level: "warn", msg: "triage_deployment_missing", tenantId, workflow: workflow.id, source: workflow.source }); - } - const placed = await deps.codeReviewPlaced(tenantId); - const reviewed = await deps.codeReviewSources(tenantId); - for (const workflow of CODE_REVIEW_WORKFLOWS) { - if (!placed) await releaseAll(tenantId, workflow.id, "no_isolated_place"); - else if (!connected.includes(workflow.source) || !reviewed.includes(workflow.source)) await releaseAll(tenantId, workflow.id, "code_review_off"); - } - for (const variant of PR_WORKFLOW_VARIANTS) { - if (!placed) await releaseAll(tenantId, variant.id, "no_isolated_place"); - else if (!connected.includes(variant.source)) await releaseAll(tenantId, variant.id, "source_disconnected"); - } - if ((await deps.liveDeployments(tenantId, triageWorkflowFor("github").id)).some((deployment) => !deployment.cancelling)) { - await releaseAll(tenantId, LEGACY_TRIAGE_WORKFLOW, "superseded_by_pr_triage_github"); } }; } diff --git a/apps/hub/src/github/tenant-config.ts b/apps/hub/src/github/tenant-config.ts index c5763ff64..1fbaffdf2 100644 --- a/apps/hub/src/github/tenant-config.ts +++ b/apps/hub/src/github/tenant-config.ts @@ -99,36 +99,6 @@ export async function patchCorbitsTriage( }); } -export type RotationRecord = { from: string; to: string; at: string }; - -export const ROTATION_KEY = "rotation"; - -/** `key` names the deployment the record is of; the GitHub one keeps the original key. */ -export function rotationRecord(ns: CorbitsTriageNs, key = ROTATION_KEY): RotationRecord | undefined { - const value = asRecord(ns[key]); - const { from, to, at } = value ?? {}; - return typeof from === "string" && typeof to === "string" && typeof at === "string" ? { from, to, at } : undefined; -} - -export async function claimRotation(db: DB["db"], tenantId: string, record: RotationRecord, key = ROTATION_KEY): Promise { - let claimed = false; - await patchCorbitsTriage(db, tenantId, function claim(ns) { - if (rotationRecord(ns, key) !== undefined) return undefined; - claimed = true; - return { ...ns, [key]: record }; - }); - return claimed; -} - -export async function clearRotation(db: DB["db"], tenantId: string, record: RotationRecord, key = ROTATION_KEY): Promise { - await patchCorbitsTriage(db, tenantId, function clear(ns) { - const current = rotationRecord(ns, key); - if (current?.from !== record.from || current.to !== record.to) return undefined; - const { [key]: _rotation, ...rest } = ns; - return rest; - }); -} - type LegacyRepoRecord = RepoRecord & { classificationAuthorized?: unknown }; function hasLegacyFlag(row: unknown): boolean { diff --git a/apps/hub/src/github/tenant-patch.postgres.test.ts b/apps/hub/src/github/tenant-patch.postgres.test.ts index 1dca868b9..aac4f96ba 100644 --- a/apps/hub/src/github/tenant-patch.postgres.test.ts +++ b/apps/hub/src/github/tenant-patch.postgres.test.ts @@ -8,7 +8,7 @@ import { createDB, createGrantStore, runMigrations, schema } from "@intx/db"; import { asRecord } from "@corbits/common"; import { bitbucketConnection } from "../bitbucket-datacenter/records.js"; import { allowCreator } from "./admin-github-grant.js"; -import { claimRotation, patchCorbitsTriage, prepareCorbitsTriagePatch, repoRecords, rotationRecord, triageNs, type CorbitsTriageNs, type RepoRecord } from "./tenant-config.js"; +import { patchCorbitsTriage, prepareCorbitsTriagePatch, repoRecords, triageNs, type CorbitsTriageNs, type RepoRecord } from "./tenant-config.js"; const TEST_DB = { host: "localhost", port: 5432, user: "postgres", password: "postgres", database: "interchange" }; const T = "tnt_patch"; @@ -121,18 +121,17 @@ test("PostgreSQL: a manager's tenant patch keeps the connection, code review and expect(await errorOf(await prepareCorbitsTriagePatch(deps, T, MANAGER, patchReq([])))).toEqual([400, "invalid_request"]); // A write landing between the hub's read and its locked write is refused, not overwritten. - const rotation = { from: "run_a", to: "run_b", at: "2026-10-18T00:00:00.000Z" }; const interleaved = { ...deps, async authorize(principalId: string, tenantId: string, resource: string, action: string) { - await claimRotation(db, T, rotation); + await patchCorbitsTriage(db, T, (ns) => ({ ...ns, interleaved: true })); return deps.authorize(principalId, tenantId, resource, action); }, }; const repinned = { ...current, repos: [{ ...github, checkPack: { name: "policy", id: "art_a" } }, bitbucket] }; expect(await errorOf(await prepareCorbitsTriagePatch(interleaved, T, MANAGER, patchReq(repinned)))).toEqual([409, "config_conflict"]); current = await storedNs(); - expect(rotationRecord(current)).toEqual(rotation); + expect(current["interleaved"]).toBe(true); const kept = repoRecords(current)[0]; expect(await errorOf(await prepareCorbitsTriagePatch(deps, T, MANAGER, new Request(`https://hub/api/tenants/${T}`, { method: "PATCH", body: JSON.stringify({ name: 7, config: { corbitsTriage: { ...current, repos: [kept] } } }) })))).toEqual([400, "invalid_request"]); diff --git a/apps/hub/src/github/triage-queue.ts b/apps/hub/src/github/triage-queue.ts index 8cbdc12f0..ffa0c87a2 100644 --- a/apps/hub/src/github/triage-queue.ts +++ b/apps/hub/src/github/triage-queue.ts @@ -6,6 +6,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import type { PrTriageRow, TriageEvent } from "@corbits/triage-contracts"; import { TriageStoppedError } from "./deployment.js"; import { BASE_MOVED, type ReconcilePolicy } from "./reconcile-plan.js"; +import type { RepoRecord } from "./tenant-config.js"; import { TriageStateConflictError, type TriageStateStore, type TriageStateVersion } from "./triage-state-store.js"; /** Why a head's mail did not go out: the next step a person takes when Triage is stopped, else the error. */ @@ -13,6 +14,17 @@ export function deliveryFailure(err: unknown): string { return err instanceof TriageStoppedError ? err.message : `delivery failed: ${String(err)}`; } +/** A pull request mail for the host's listener. */ +export type TriageJob = { + tenantId: string; + repo: string; + number: number; + headSha: string; + payload: unknown; + /** The repository a triage job's mail names, which the hub reads the pull request through. */ + record?: RepoRecord; +}; + export type QueuedHead = { tenantId: string; repo: string; number: number; headSha: string }; export type Remembered = { prior: PrTriageRow | undefined; rows: PrTriageRow[]; version: TriageStateVersion }; diff --git a/apps/hub/src/github/triage-reconciler.ts b/apps/hub/src/github/triage-reconciler.ts index b283038cd..b7dcb18a6 100644 --- a/apps/hub/src/github/triage-reconciler.ts +++ b/apps/hub/src/github/triage-reconciler.ts @@ -7,9 +7,8 @@ // out, on hosts and pull requests whose CI results send no webhook. import { dependsOnCi, hasEnabledRules, marksWaiting, repoPolicy, type PrTriageRow, type RepoPolicy, type RepoSource, type VerdictDetail } from "@corbits/triage-contracts"; import { mailPayload, unreadablePayload, waitingPayload, type PolicyUnreadable } from "./bridge.js"; -import type { TriageJob } from "./coordinator.js"; +import type { TriageJob } from "./triage-queue.js"; import { GithubRateLimited } from "./github-usage.js"; -import { triageWorkflowFor } from "./source-workflow.js"; import { deliveryFailure, failHead } from "./triage-queue.js"; import type { PolicyReader, StoredPack } from "./check-pack-store.js"; import { inFlight, planTenant, requeue, runKey, type OpenPr, type QueuedPr, type ReconcilePolicy, type RepoPlan } from "./reconcile-plan.js"; @@ -267,7 +266,7 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { if (!("pack" in repo) || !marksWaiting(sourceOf(repo.record), repo.automatic)) return; const waited = new Set(repo.stored.rows.filter((row) => row.status === "waiting").map(headKey)); for (const row of rows.filter((candidate) => candidate.status === "waiting" && !waited.has(headKey(candidate)))) { - await deps.send({ tenantId: pass.tenantId, workflow: triageWorkflowFor(deps.source).id, repo: triageStateKey(repo.record), number: row.number, headSha: row.headSha, source: "reconcile", payload: waitingPayload(repo.record, repo.policy, repo, row, false) }); + await deps.send({ tenantId: pass.tenantId, repo: triageStateKey(repo.record), number: row.number, headSha: row.headSha, payload: waitingPayload(repo.record, repo.policy, repo, row, false) }); } } @@ -286,7 +285,7 @@ export function createTriageReconciler(deps: TriageReconcilerDeps) { const mailed = { ...extra, ...(queued.noWait && { noWait: true }) }; const payload = "pack" in repo ? mailPayload(repo.record, repo.policy, repo, mailed) : unreadablePayload(repo.record, repo.policy, repo, mailed); try { - await deps.send({ tenantId: pass.tenantId, workflow: triageWorkflowFor(deps.source).id, repo: key, number: queued.number, headSha, source: "reconcile", record: repo.record, payload }); + await deps.send({ tenantId: pass.tenantId, repo: key, number: queued.number, headSha, record: repo.record, payload }); } catch (err) { if (err instanceof GithubRateLimited) throw err; deps.log({ level: "error", msg: "forward_failed", tenantId: pass.tenantId, repo: repo.record.name, pr: queued.number, headSha, error: String(err) }); diff --git a/apps/hub/src/github/worker-spawn.postgres.test.ts b/apps/hub/src/github/worker-spawn.postgres.test.ts deleted file mode 100644 index 08cf8a95b..000000000 --- a/apps/hub/src/github/worker-spawn.postgres.test.ts +++ /dev/null @@ -1,345 +0,0 @@ -// Requires PostgreSQL at TEST_DB. A triage job through the coordinator and the -// real worker spawner: a missing listener and a provisioner's capacity decline -// defer the job, a worker still starting is polled until it takes its mail, and -// under the hub's lifecycle policy the stock sweep releases a finished worker's -// sidecar as soon as its run ends. -import { expect, test } from "bun:test"; -import { randomBytes } from "node:crypto"; -import { eq, sql } from "drizzle-orm"; -import { createDB, createWorkflowRunLaunchSpecStore, runMigrations, schema, type DB } from "@intx/db"; -import { createWorkflowHistoryReceiveTracker, createWorkflowLifecycleService, WorkflowProvisioningError } from "@intx/hub-sessions"; -import type { ResolvedWorkflowLifecyclePolicy } from "@intx/types"; -import { interchangeSettings, type HubEnv } from "../env.js"; -import { CAPACITY_UNAVAILABLE } from "../sidecar/host-capacity.js"; -import { createCoordinator, NO_LISTENER_REASON, type TriageJob } from "./coordinator.js"; -import { DeploymentNotReadyError, SenderRefusedError } from "./deployment.js"; -import { createWorkerSpawner } from "./worker-spawn.js"; - -const TEST_DB = { host: "localhost", port: 5432, user: "postgres", password: "postgres", database: "interchange" }; -const TENANT = "tnt_spawn"; -const DOMAIN = "spawn.test"; -const RUN_TIMEOUT_MS = 60 * 60_000; -const WORKFLOW = "pr-triage"; -const BUNDLE = { - source: { kind: "asset", assetId: "ast_triage", package: { format: "source", commitSha: "c0ffee", packageName: "@corbits/pr-triage-workflow" } }, - entry: "./pr-triage.mjs", - projection: { id: WORKFLOW, stepOrder: ["a"], steps: { a: { kind: "step", id: "a" } }, triggers: [] }, - closure: { schemaVersion: "1", topLevel: [], entries: [] }, - approvedWireHash: "deadbeef", - approvedGrants: [], -}; -const JOB: TriageJob = { tenantId: TENANT, workflow: WORKFLOW, repo: "acme/widgets", number: 8, headSha: "abc123", source: "event", payload: { prNumber: 8 } }; - -async function withDatabase(body: (db: DB["db"]) => Promise): Promise { - const database = `corbits_spawn_${randomBytes(6).toString("hex")}`; - const admin = createDB(TEST_DB); - await admin.db.execute(sql.raw(`create database ${database}`)); - const config = { ...TEST_DB, database }; - let handle: ReturnType | undefined; - try { - await runMigrations(config, { schema: "public" }); - handle = createDB(config); - const { db } = handle; - await db.insert(schema.tenant).values({ id: TENANT, name: "Spawn", slug: "spawn", domain: DOMAIN, config: {} }); - await db.insert(schema.principal).values({ id: "prn_deployer", tenantId: TENANT, kind: "user", refId: "deployer", status: "active" }); - await db.insert(schema.workflowDefinition).values({ id: "wfd_triage", tenantId: TENANT, name: WORKFLOW }); - await body(db); - } finally { - await handle?.close(); - await admin.db.execute(sql.raw(`drop database if exists ${database} with (force)`)); - await admin.close(); - } -} - -async function deployWorker(db: DB["db"], runId: string, content: Record, lifecyclePolicy?: ResolvedWorkflowLifecyclePolicy): Promise { - await db.insert(schema.workflowRun).values({ id: runId, anchorRunId: runId, definitionId: "wfd_triage", tenantId: TENANT, status: "deployed", lifecyclePolicy }); - await createWorkflowRunLaunchSpecStore(db).create({ - anchorRunId: runId, - sessionId: `ses_${runId}`, - deploymentDomain: DOMAIN, - sourceAuthorityPrincipalId: "prn_deployer", - frozenApprovalBundle: BUNDLE, - sourceOfferingIds: ["off_a"], - defaultSourceOfferingId: "off_a", - deployContent: { systemPrompt: "Triage pull requests.", ...content }, - }); -} - -function notReached(): never { - throw new Error("only reached with pending history or a live sidecar"); -} - -async function nothingToReady(): Promise {} - -test("PostgreSQL: a job with no listener to copy, then refused for capacity, is deferred both times and its worker is fired once it starts", async () => { - await withDatabase(async function deferThenFire(db) { - // The clock moves only when a timer fires, so a deferred job always waits for its timer. - let clock = 0; - let timer = Promise.withResolvers<() => Promise>(); - const released = Promise.withResolvers(); - const delivered: string[] = []; - const failed: string[] = []; - let refusals = 1; - let notReady = 1; - const coordinator = createCoordinator({ - spawner: createWorkerSpawner({ - db, - beforeDeploy: nothingToReady, - async redeploy(_tenantId, _seed, content) { - if (refusals-- > 0) throw new WorkflowProvisioningError(CAPACITY_UNAVAILABLE, "424 MiB free, 512 MiB needed"); - await deployWorker(db, "run_worker", content); - return { status: "deployed", runId: "run_worker" }; - }, - async deliver(_tenantId, address) { - if (notReady-- > 0) throw new DeploymentNotReadyError(); - delivered.push(address); - }, - lifecycle: { - async requestCancellation(_tenantId, runId) { - released.resolve(runId); - return "pending"; - }, - getStatus: async () => null, - }, - sleep: () => Bun.sleep(1), - fireTimeoutMs: 60_000, - pollMs: 1, - }), - retry: { minMs: 1, maxMs: 1 }, - async failed(job, error) { - failed.push(`${job.number}: ${error}`); - }, - runTimeoutMs: RUN_TIMEOUT_MS, - setTimer(fire, ms) { - if (ms === RUN_TIMEOUT_MS) return () => {}; - timer.resolve(async function elapse() { - clock += ms; - await fire(); - }); - return () => {}; - }, - now: () => new Date(clock), - log: () => {}, - }); - - coordinator.enqueue(JOB); - const noListener = await timer.promise; - - timer = Promise.withResolvers(); - await db.insert(schema.workflowRun).values({ id: "run_listener", anchorRunId: "run_listener", definitionId: "wfd_triage", tenantId: TENANT, status: "deployed" }); - await noListener(); - const noCapacity = await timer.promise; - expect(refusals).toBe(0); - - await noCapacity(); - expect(await released.promise).toBe("run_worker"); - expect(delivered).toEqual([`run_worker@${DOMAIN}`]); - expect(failed).toEqual([]); - coordinator.stop(); - }); -}, 120_000); - -test("PostgreSQL: a job with no listener to copy is retried five times, then its head fails and it is not retried again", async () => { - await withDatabase(async function deferThenFail(db) { - let clock = 0; - let timer: (() => Promise) | undefined; - let settled = Promise.withResolvers(); - const failed: string[] = []; - const coordinator = createCoordinator({ - spawner: createWorkerSpawner({ - db, - beforeDeploy: nothingToReady, - redeploy: notReached, - deliver: notReached, - lifecycle: { requestCancellation: notReached, getStatus: notReached }, - sleep: () => Bun.sleep(1), - fireTimeoutMs: 60_000, - pollMs: 1, - }), - retry: { minMs: 1, maxMs: 1 }, - async failed(job, error) { - failed.push(`${job.number}: ${error}`); - }, - runTimeoutMs: RUN_TIMEOUT_MS, - setTimer(fire, ms) { - if (ms === RUN_TIMEOUT_MS) return () => {}; - timer = async function elapse() { - clock += ms; - await fire(); - }; - return () => {}; - }, - now: () => new Date(clock), - log(entry) { - settled.resolve(String(entry.msg)); - }, - }); - - const outcomes: string[] = []; - coordinator.enqueue(JOB); - // Bounded so a job retried forever fails the test instead of hanging it. - while (outcomes.length < 10) { - outcomes.push(await settled.promise); - settled = Promise.withResolvers(); - const elapse = timer; - timer = undefined; - if (!elapse) break; - await elapse(); - } - expect(outcomes).toEqual([...Array(5).fill("worker_spawn_deferred"), "worker_deployment_unavailable"]); - expect(failed).toEqual([`8: ${NO_LISTENER_REASON}`]); - coordinator.stop(); - }); -}, 120_000); - -test("PostgreSQL: a worker whose first mail the sender gate refuses is released and its job deferred, and the next worker takes the mail", async () => { - await withDatabase(async function refusedThenFired(db) { - let clock = 0; - const timer = Promise.withResolvers<() => Promise>(); - const released: string[] = []; - const delivered: string[] = []; - const failed: string[] = []; - let spawned = 0; - await db.insert(schema.workflowRun).values({ id: "run_listener", anchorRunId: "run_listener", definitionId: "wfd_triage", tenantId: TENANT, status: "deployed" }); - const fired = Promise.withResolvers(); - const coordinator = createCoordinator({ - spawner: createWorkerSpawner({ - db, - beforeDeploy: nothingToReady, - async redeploy(_tenantId, _seed, content) { - const runId = `run_worker_${++spawned}`; - await deployWorker(db, runId, content); - return { status: "deployed", runId }; - }, - async deliver(_tenantId, address) { - if (spawned === 1) throw new SenderRefusedError("Workflow run does not admit mail from this sender"); - delivered.push(address); - fired.resolve(); - }, - lifecycle: { - async requestCancellation(_tenantId, runId) { - released.push(runId); - return "pending"; - }, - getStatus: async () => null, - }, - sleep: () => Bun.sleep(1), - fireTimeoutMs: 60_000, - pollMs: 1, - }), - retry: { minMs: 1, maxMs: 1 }, - async failed(job, error) { - failed.push(`${job.number}: ${error}`); - }, - runTimeoutMs: RUN_TIMEOUT_MS, - setTimer(fire, ms) { - if (ms === RUN_TIMEOUT_MS) return () => {}; - timer.resolve(async function elapse() { - clock += ms; - await fire(); - }); - return () => {}; - }, - now: () => new Date(clock), - log: () => {}, - }); - - coordinator.enqueue(JOB); - const refused = await timer.promise; - expect(released).toEqual(["run_worker_1"]); - expect(failed).toEqual([]); - - await refused(); - await fired.promise; - expect(delivered).toEqual([`run_worker_2@${DOMAIN}`]); - expect(failed).toEqual([]); - coordinator.stop(); - }); -}, 120_000); - -test("PostgreSQL: under the hub's lifecycle policy, a worker's sidecar is released as soon as its run ends", async () => { - await withDatabase(async function releaseOnEnd(db) { - const { defaultLifecyclePolicy } = interchangeSettings({ PORT: "0", HUB_DATA_DIR: "unused", PRINCIPAL_KEY_ENCRYPTION_KEY: "0".repeat(64) } as HubEnv); - await db.insert(schema.workflowRun).values({ id: "run_listener", anchorRunId: "run_listener", definitionId: "wfd_triage", tenantId: TENANT, status: "deployed" }); - const lifecycle = createWorkflowLifecycleService({ - db, - retentionRouter: { - getRetentionCandidates: () => [], - fenceAllocation() {}, - retainAllocation: notReached, - getAllocationRetention: notReached, - holdsAllocatedBinding: notReached, - retireAllocation: notReached, - }, - runReader: { - listRunIds: notReached, - readRunEvents: notReached, - readLatestRunEvents: notReached, - resolveRefTip: notReached, - hasRepository: notReached, - }, - historyReceives: createWorkflowHistoryReceiveTracker(), - }); - const errors: Array> = []; - const coordinator = createCoordinator({ - spawner: createWorkerSpawner({ - db, - beforeDeploy: nothingToReady, - async redeploy(_tenantId, _seed, content) { - await deployWorker(db, "run_worker", content, defaultLifecyclePolicy); - await db.insert(schema.sidecarAllocation).values({ - id: "alc_worker", - anchorRunId: "run_worker", - tenantId: TENANT, - provisionerId: "docker", - provisionerApiVersion: 2, - provisionerBindingFingerprint: "fp", - adapterManifestDigest: "digest", - status: "allocated", - maxDisconnectedMs: 60_000, - }); - return { status: "deployed", runId: "run_worker" }; - }, - async deliver() { - await db.insert(schema.workflowRun).values({ id: "run_body", anchorRunId: "run_worker", definitionId: "wfd_triage", tenantId: TENANT, status: "completed" }); - }, - lifecycle, - sleep: () => Bun.sleep(5), - fireTimeoutMs: 60_000, - pollMs: 5, - }), - retry: { minMs: 1, maxMs: 1 }, - failed: async () => {}, - runTimeoutMs: RUN_TIMEOUT_MS, - setTimer: () => () => {}, - now: () => new Date(), - log: (entry) => entry.level === "error" && errors.push(entry), - }); - - coordinator.enqueue(JOB); - while ((await workerRun(db)).cancellationRequestedAt === null) await Bun.sleep(5); - // Stock stops the idle anchor once its cancel grace runs out. - const endedAt = Date.now(); - await db.update(schema.workflowRun).set({ status: "cancelled", endedAt: new Date(endedAt) }).where(eq(schema.workflowRun.id, "run_worker")); - let allocation = await workerAllocation(db); - while (allocation.status === "allocated" && Date.now() - endedAt < 1_000) { - await lifecycle.reconcileNext(); - allocation = await workerAllocation(db); - } - expect(allocation.status).toBe("releasing"); - expect(errors).toEqual([]); - coordinator.stop(); - }); -}, 120_000); - -async function workerRun(db: DB["db"]): Promise { - const [run] = await db.select().from(schema.workflowRun).where(eq(schema.workflowRun.id, "run_worker")); - if (!run) throw new Error("worker run missing"); - return run; -} - -async function workerAllocation(db: DB["db"]): Promise { - const [allocation] = await db.select().from(schema.sidecarAllocation).where(eq(schema.sidecarAllocation.id, "alc_worker")); - if (!allocation) throw new Error("worker allocation missing"); - return allocation; -} diff --git a/apps/hub/src/github/worker-spawn.ts b/apps/hub/src/github/worker-spawn.ts deleted file mode 100644 index 5de94c8cc..000000000 --- a/apps/hub/src/github/worker-spawn.ts +++ /dev/null @@ -1,106 +0,0 @@ -// Spawns one worker deployment per triage job: a fresh copy of the host's -// live pr-triage listener (the seed), made the way rotation makes its copy and -// marked a worker in its deploy content, fired with exactly one mail. The -// worker keeps the listener shape, so its anchor stays live while the body -// run's log lands: the hub refuses a run log pushed after its anchor ended, -// which a one-run top-level deployment always does. The deployment is -// cancelled once that one body run has ended. -import { and, eq, ne, notInArray } from "drizzle-orm"; -import { schema, type DB } from "@intx/db"; -import { WorkflowProvisioningError, type WorkflowLifecycleService } from "@intx/hub-sessions"; -import { CAPACITY_UNAVAILABLE } from "../sidecar/host-capacity.js"; -import { NO_LISTENER_REASON, WorkerDeferredError, WorkerOutdatedError, type WorkerSpawner, type TriageJob } from "./coordinator.js"; -import { DeploymentNotReadyError, liveWorkers, SenderRefusedError, resolveLiveDeployment, resolveLiveDeployments, WORKER_MARKER, type Redeployed } from "./deployment.js"; - -const WORKER_RELEASE_REASON = "Triage worker finished its run"; -const RESTART_RELEASE_REASON = "Triage worker orphaned by a hub restart"; - -export type WorkerSpawnDeps = { - db: DB["db"]; - /** Readies what the worker's credentials hold: a worker is deployed with them as they are. */ - beforeDeploy: (job: TriageJob) => Promise; - redeploy: (tenantId: string, seedAnchorRunId: string, content: Record) => Promise; - deliver: (tenantId: string, address: string, payload: unknown) => Promise; - lifecycle: Pick; - sleep: (ms: number) => Promise; - /** How long a fresh worker may take to accept its mail; the stock first-connect window is two minutes. */ - fireTimeoutMs: number; - pollMs: number; -}; - -export function createWorkerSpawner(deps: WorkerSpawnDeps): WorkerSpawner { - async function fire(job: TriageJob, runId: string): Promise { - const deadline = Date.now() + deps.fireTimeoutMs; - for (;;) { - const [worker] = (await resolveLiveDeployments(deps.db, job.tenantId, job.workflow, "worker")).filter((live) => live.runId === runId); - try { - if (!worker) throw new DeploymentNotReadyError(); - await deps.deliver(job.tenantId, worker.address, job.payload); - return; - } catch (err) { - if (!(err instanceof DeploymentNotReadyError) || Date.now() >= deadline) throw err; - } - await deps.sleep(deps.pollMs); - } - } - - async function spawn(job: TriageJob): Promise { - const seed = await resolveLiveDeployment(deps.db, job.tenantId, job.workflow); - // The listener is being deployed or healed; the reconciler sees to it. - if (!seed) throw new WorkerDeferredError(`no live ${job.workflow} listener to copy`, NO_LISTENER_REASON); - await deps.beforeDeploy(job); - let copy: Redeployed; - try { - copy = await deps.redeploy(job.tenantId, seed.runId, { [WORKER_MARKER]: true }); - } catch (err) { - if (err instanceof WorkflowProvisioningError && err.code === CAPACITY_UNAVAILABLE) throw new WorkerDeferredError(err.message); - throw err; - } - if (copy.status !== "deployed") throw new Error(`worker not deployed: ${copy.reason}`); - try { - await fire(job, copy.runId); - } catch (err) { - await deps.lifecycle.requestCancellation(job.tenantId, copy.runId, `Triage worker could not be fired: ${String(err)}`); - if (err instanceof SenderRefusedError) throw new WorkerOutdatedError(err.message); - throw err; - } - return copy.runId; - } - - async function bodyEnded(runId: string): Promise { - const { workflowRun, liveWorkflowRunStatuses } = schema; - const [row] = await deps.db - .select({ id: workflowRun.id }) - .from(workflowRun) - .where(and(eq(workflowRun.anchorRunId, runId), ne(workflowRun.id, runId), notInArray(workflowRun.status, [...liveWorkflowRunStatuses]))) - .limit(1); - return row !== undefined; - } - - async function anchorEnded(tenantId: string, runId: string): Promise { - const status = await deps.lifecycle.getStatus(tenantId, runId); - return status === null || !schema.isLiveWorkflowRunStatus(status.status); - } - - async function ended(tenantId: string, runId: string): Promise { - while (!(await bodyEnded(runId)) && !(await anchorEnded(tenantId, runId))) await deps.sleep(deps.pollMs); - } - - async function release(tenantId: string, runId: string): Promise { - await deps.lifecycle.requestCancellation(tenantId, runId, WORKER_RELEASE_REASON); - } - - return { spawn, ended, release }; -} - -/** A hub restart loses the queue that was waiting on its workers, so every worker still live is released. */ -export async function releaseOrphanedWorkers(db: DB["db"], lifecycle: Pick, log: (entry: Record) => void): Promise { - for (const worker of await liveWorkers(db)) { - try { - await lifecycle.requestCancellation(worker.tenantId, worker.runId, RESTART_RELEASE_REASON); - log({ level: "info", msg: "worker_released_on_start", ...worker }); - } catch (err) { - log({ level: "error", msg: "worker_release_failed", ...worker, error: String(err) }); - } - } -} diff --git a/apps/hub/src/integrations/app.ts b/apps/hub/src/integrations/app.ts index db9c9f85b..d5572bf94 100644 --- a/apps/hub/src/integrations/app.ts +++ b/apps/hub/src/integrations/app.ts @@ -665,7 +665,7 @@ export function createIntegrationsApp(guards: IntegrationGuards, handlers: Integ `${WORKFLOW_VERSIONS_PATH}/:tenantId`, describeRoute({ summary: "Package versions of the workspace's live workflow deployments", - description: "Each live deployment, newest first, with the workflow package it pinned and the packages its frozen closure carries. `package` is null for a deployment not pinned from a package registry.", + description: "Each live deployment, newest first, with the workflow package it pinned and the packages its frozen closure carries. `package` is null for a deployment not pinned from a package registry; `source` is the asset and commit a source-format deployment runs, else null.", tags: TAGS, responses: { 200: json("Live deployments and their versions", DeployedVersions), 401: failures[401], 403: failures[403] }, }), diff --git a/apps/hub/src/manager-intake.postgres.test.ts b/apps/hub/src/manager-intake.postgres.test.ts index 9b3236d07..c75b89172 100644 --- a/apps/hub/src/manager-intake.postgres.test.ts +++ b/apps/hub/src/manager-intake.postgres.test.ts @@ -4,7 +4,7 @@ import { randomBytes } from "node:crypto"; import { sql } from "drizzle-orm"; import { createDB, runMigrations, schema } from "@intx/db"; import { APPROVE_DO_KIND, emptyPack, repoPolicy } from "@corbits/triage-contracts"; -import type { PullRequestToReview } from "./code-review-feed.js"; +import type { PullRequestToReview } from "./pull-request-reader.js"; import { mailPayload, waitingPayload } from "./github/bridge.js"; import { createPauseGate } from "./github/pr-workflow.js"; import { createManagerIntake } from "./manager-intake.js"; @@ -45,14 +45,16 @@ async function passed(_tenantId: string, _record: unknown, _number: number, requ return request; } -function stack(db: Db, listener: string | undefined, pull = PULL, gate: Gate = passed) { +const OPEN = [{ number: 7, headSha: "h1", draft: false }, { number: 9, headSha: "h9", draft: true }]; + +function stack(db: Db, listener: string | undefined, pull = PULL, gate: Gate = passed, open = OPEN) { const mailed: { address: string; payload: unknown }[] = []; const logged: Record[] = []; const toListener = createManagerIntake({ db, listener: async () => listener, readPull: async () => pull, - openHeads: async () => [{ number: 7, headSha: "h1", draft: false }, { number: 9, headSha: "h9", draft: true }], + openHeads: async () => open, withAppLogin: async (_tenantId, payload) => ({ ...payload, appLogin: "triage[bot]" }), async deliver(_tenantId, address, payload) { mailed.push({ address, payload }); @@ -177,3 +179,20 @@ test("PostgreSQL: every pull request a mail names passes the gate, whatever sent expect(mailed.flatMap((mail) => (mail.payload as { items: { prNumber: number }[] }).items.map((item) => item.prNumber))).toEqual([7, 7]); }); }, 120_000); + +test("PostgreSQL: a backlog with no open pull request is taken with nothing recorded, and a failing gate read still leaves the heads recorded before it mailed", async () => { + await withDatabase(async function partial(db) { + const drafts = stack(db, LISTENER, PULL, passed, [{ number: 9, headSha: "h9", draft: true }]); + expect(await drafts.toListener(TENANT, { ...mailPayload(RECORD, POLICY, STORED, {}), kind: "backlog" })).toBe(true); + expect(drafts.mailed).toEqual([]); + + const { toListener, mailed } = stack(db, LISTENER, PULL, async function failOn9(_tenantId, _record, number, request) { + if (number === 9) throw new Error("head read failed"); + return request; + }); + const catchUp = mailPayload(RECORD, POLICY, STORED, { items: [{ prNumber: 7, headSha: "h1" }, { prNumber: 9, headSha: "h9" }] }); + await expect(toListener(TENANT, catchUp)).rejects.toThrow("head read failed"); + expect((await events(db)).map((row) => row.state.head)).toEqual(["h1"]); + expect(mailed.map((mail) => (mail.payload as { items: { prNumber: number }[] }).items.map((item) => item.prNumber))).toEqual([[7]]); + }); +}, 120_000); diff --git a/apps/hub/src/manager-intake.ts b/apps/hub/src/manager-intake.ts index f9d1aa6d1..5f3acfb67 100644 --- a/apps/hub/src/manager-intake.ts +++ b/apps/hub/src/manager-intake.ts @@ -7,7 +7,7 @@ import { eq } from "drizzle-orm"; import { schema, type DB } from "@intx/db"; import { asRecord } from "@corbits/common"; import { APPROVE_DO_KIND, repoSourceOf, WAITING_KIND, type RepoSource } from "@corbits/triage-contracts"; -import { reviewRequestOf, type PullRequestToReview } from "./code-review-feed.js"; +import { reviewRequestOf, type PullRequestToReview } from "./pull-request-reader.js"; import { repoRecords, sourceOf, triageNs, type RepoRecord } from "./github/tenant-config.js"; import { liveManager, recordEvent } from "./manager-state.js"; @@ -94,8 +94,9 @@ export function createManagerIntake(deps: ManagerIntakeDeps) { /** * Records each pull request a mail names. One whose manager is up is mailed that run; the rest go to the listener, - * which deploys them. False, with nothing recorded, when the workspace has no listener or the mail is not one a - * manager runs. + * which deploys them, including those recorded before a later one failed. True with nothing recorded when the mail + * names no pull request, as a backlog of a repository with none open; false when the workspace has no listener or + * the mail is not one a manager runs. */ return async function toListener(tenantId: string, payload: unknown): Promise { const request = asRecord(payload); @@ -106,24 +107,26 @@ export function createManagerIntake(deps: ManagerIntakeDeps) { if (!address || typeof repo !== "string") return false; const record = await recordOf(tenantId, repo, source); const pulls = await pullsOf(tenantId, record, request); - if (pulls.length === 0) return false; const domain = domainOf(address); const start: Named[] = []; - for (const asked of pulls) { - const request = await deps.gate(tenantId, record, asked.number, asked.request); - if (!request) continue; - const pull = { ...asked, request }; - const recorded = await deps.withAppLogin(tenantId, await withReviewRequest(tenantId, record, pull)); - await recordEvent(deps.db, tenantId, { source, repo, number: pull.number }, pull.id, { type: "webhook", head: pull.head, payload: recorded as Record }); - const runId = await liveManager(deps.db, tenantId, { source, repo, number: pull.number }); - if (runId) { - await deps.deliver(tenantId, `${runId}@${domain}`, { kind: "manage", tenantId, repo, repoSource: source, prNumber: pull.number }); - } else { - start.push(pull); + try { + for (const asked of pulls) { + const request = await deps.gate(tenantId, record, asked.number, asked.request); + if (!request) continue; + const pull = { ...asked, request }; + const recorded = await deps.withAppLogin(tenantId, await withReviewRequest(tenantId, record, pull)); + await recordEvent(deps.db, tenantId, { source, repo, number: pull.number }, pull.id, { type: "webhook", head: pull.head, payload: recorded as Record }); + const runId = await liveManager(deps.db, tenantId, { source, repo, number: pull.number }); + if (runId) { + await deps.deliver(tenantId, `${runId}@${domain}`, { kind: "manage", tenantId, repo, repoSource: source, prNumber: pull.number }); + } else { + start.push(pull); + } + } + } finally { + if (start.length > 0) { + await deps.deliver(tenantId, address, { kind: "pr", repo, ...(source !== "github" && { repoSource: source }), tenantId, items: start.map((pull) => ({ prNumber: pull.number, headSha: pull.head })) }); } - } - if (start.length > 0) { - await deps.deliver(tenantId, address, { kind: "pr", repo, ...(source !== "github" && { repoSource: source }), tenantId, items: start.map((pull) => ({ prNumber: pull.number, headSha: pull.head })) }); } return true; }; diff --git a/apps/hub/src/provider-hooks/intake.ts b/apps/hub/src/provider-hooks/intake.ts index 9948059ab..69e81b28e 100644 --- a/apps/hub/src/provider-hooks/intake.ts +++ b/apps/hub/src/provider-hooks/intake.ts @@ -28,7 +28,7 @@ export type ProviderHook = { ownUser?: (ns: CorbitsTriageNs) => string | undefined; }; -export type ProviderHookDeps = Pick; +export type ProviderHookDeps = Pick; type Delivery = { hook: BridgeHook; delivery: string; event: string; record: RepoRecord; ns: CorbitsTriageNs }; @@ -183,7 +183,6 @@ export function createProviderHookHandler(d: ProviderHookDeps) { if (intake.branches && (record.defaultBranch === undefined || !intake.branches.includes(record.defaultBranch))) return ignored(at, "not_default_branch", { branches: intake.branches }); return policyPush(at, intake.push); } - if (intake.kind === "pr") void d.codeReview.offer(hook.tenantId, record, intake.mail); if (!repoPolicy(record).enabled) return ignored(at, "repo_not_enabled"); if (intake.kind === "base-move") return baseMove(at, intake.move); return pullRequest(at, intake.mail, provider); diff --git a/apps/hub/src/code-review-feed.test.ts b/apps/hub/src/pull-request-reader.test.ts similarity index 51% rename from apps/hub/src/code-review-feed.test.ts rename to apps/hub/src/pull-request-reader.test.ts index abf5e3eb5..9f97cf60d 100644 --- a/apps/hub/src/code-review-feed.test.ts +++ b/apps/hub/src/pull-request-reader.test.ts @@ -1,70 +1,16 @@ import { expect, test } from "bun:test"; -import type { RepoSource } from "@corbits/triage-contracts"; -import { createCodeReviewFeed, createPullRequestReader, cutDiff, diffHash, type PullRequestToReview } from "./code-review-feed.js"; -import type { TriageJob } from "./github/coordinator.js"; -import type { PrMail } from "./github/normalize.js"; +import { createPullRequestReader, cutDiff, diffHash, reviewRequestOf, type PullRequestToReview } from "./pull-request-reader.js"; import type { RepoRecord } from "./github/tenant-config.js"; const RECORD: RepoRecord = { name: "acme/widgets", connected: true, codeReview: true }; const PULL: PullRequestToReview = { title: "Fix", description: "Body", headSha: "sha1", draft: false, open: true, diff: "diff --git a/x b/x\nindex 1111111..2222222 100644\n+1\n" }; -function mail(action: string, fields: Partial = {}): PrMail { - return { kind: "pr", repo: RECORD.name, prNumber: 7, deliveryId: "d", event: "pull_request", action, headSha: "sha1", author: "a", title: "Fix", body: "Body", draft: false, review: null, ...fields }; -} - -function harness(reviewed: RepoSource[] = ["github"], pull = PULL, carries = false) { - const sent: TriageJob[] = []; - const feed = createCodeReviewFeed({ - reviewed: async () => reviewed, - read: async () => pull, - carries: async () => carries, - async send(job) { - sent.push(job); - }, - log() {}, - }); - return { feed, sent }; -} - -test("each new non-draft head is sent to its host's code review once; Review again sends it anyway", async () => { - const { feed, sent } = harness(); - await feed.offer("t", RECORD, mail("synchronize")); - await feed.offer("t", RECORD, mail("synchronize")); - await feed.offer("t", RECORD, mail("labeled")); - await feed.offer("t", RECORD, mail("opened", { draft: true })); - expect(sent).toEqual([{ - tenantId: "t", - workflow: "pr-code-review-github", - repo: "acme/widgets", - number: 7, - headSha: "sha1", - source: "event", - payload: { kind: "code-review", number: 7, title: "Fix", description: "Body", diff: PULL.diff, diffHash: diffHash(PULL.diff), checkout: { source: "github", repo: "acme/widgets", sha: "sha1" } }, - }]); -}); - -test("a head that takes the newest review over is not sent", async () => { - const { feed, sent } = harness(["github"], PULL, true); - await feed.offer("t", RECORD, mail("synchronize")); - expect(sent).toEqual([]); -}); - -test("a diff's hash changes with any blob id, and a diff with none, as GitHub rebuilds from the file list, has none and is sent unhashed", async () => { +test("a diff's hash changes with any blob id, and a diff with none, as GitHub rebuilds from the file list, has none and is requested unhashed", () => { const binary = (blob: string) => `diff --git a/logo.png b/logo.png\nindex 5555555..${blob} 100644\nBinary files a/logo.png and b/logo.png differ\n`; expect(diffHash(binary("6666666"))).not.toBe(diffHash(binary("7777777"))); const rebuilt = { ...PULL, diff: "diff --git a/x b/x\n--- a/x\n+++ b/x\n@@ -1 +1 @@\n-a\n+b\n" }; expect(diffHash(rebuilt.diff)).toBeUndefined(); - const { feed, sent } = harness(["github"], rebuilt); - await feed.offer("t", RECORD, mail("synchronize")); - expect(sent[0]?.payload).not.toHaveProperty("diffHash"); -}); - -test("nothing is sent while code review is off, the key is missing or the pull request turned draft", async () => { - for (const [reviewed, record, pull] of [[[], RECORD, PULL], [["github"], { ...RECORD, codeReview: false }, PULL], [["github"], RECORD, { ...PULL, draft: true }]] as const) { - const { feed, sent } = harness([...reviewed], pull); - await feed.offer("t", record, mail("synchronize")); - expect(sent).toEqual([]); - } + expect(reviewRequestOf(RECORD, 7, rebuilt)).not.toHaveProperty("diffHash"); }); test("a diff over the limit is cut at a line boundary and names the files it cut part-way or left out", () => { @@ -91,37 +37,6 @@ test("a cut names files as git writes them with or without prefixes and quotes, expect(cutDiff(`${diff}diff --git a/p q b/r s\n`, sections[0]!.length).cut).toEqual({ lines: 6, of: 37 }); }); -test("a host answer that is not a unified diff is refused, not reviewed", async () => { - const { feed, sent } = harness(["github"], { ...PULL, diff: "{\"url\":\"…\"}" }); - await feed.offer("t", RECORD, mail("synchronize")); - expect(sent).toEqual([]); -}); - -test("two deliveries of one push in flight together send it once, and a head whose worker failed is sent again", async () => { - const sent: TriageJob[] = []; - let release = () => {}; - const held = new Promise((resolve) => (release = resolve)); - const feed = createCodeReviewFeed({ - reviewed: async () => ["github"], - read: async () => PULL, - carries: async () => false, - async send(job) { - sent.push(job); - await held; - }, - log() {}, - }); - const first = feed.offer("t", RECORD, mail("synchronize", { headSha: null })); - const second = feed.offer("t", RECORD, mail("synchronize", { headSha: null })); - release(); - await Promise.all([first, second]); - expect(sent).toHaveLength(1); - - feed.forget(sent[0]!); - await feed.offer("t", RECORD, mail("synchronize")); - expect(sent).toHaveLength(2); -}); - test("code review reads a GitHub description as GitHub shows it, without hidden text", async () => { const diff = "diff --git a/x b/x\n+1\n"; async function github(_path: string, init?: RequestInit): Promise { @@ -136,3 +51,7 @@ test("code review reads a GitHub description as GitHub shows it, without hidden expect(await read.withDiff("t", RECORD, 7)).toEqual({ title: "Fix", description: "Fixes it.", headSha: "sha1", draft: false, open: true, diff, labels: [] }); expect(await read.head("t", RECORD, 7)).toEqual({ title: "Fix", description: "Fixes it.", headSha: "sha1", draft: false, open: true, labels: [] }); }); + +test("a host answer that is not a unified diff is refused, not reviewed", () => { + expect(() => reviewRequestOf(RECORD, 7, { ...PULL, diff: "{\"url\":\"…\"}" })).toThrow("no unified diff"); +}); diff --git a/apps/hub/src/code-review-feed.ts b/apps/hub/src/pull-request-reader.ts similarity index 53% rename from apps/hub/src/code-review-feed.ts rename to apps/hub/src/pull-request-reader.ts index d63242104..a236435d1 100644 --- a/apps/hub/src/code-review-feed.ts +++ b/apps/hub/src/pull-request-reader.ts @@ -1,49 +1,26 @@ -// Feeds code review: each new non-draft head of a pull request in a repository with code review on is mailed to its -// host's pr-code-review once, through the same send path as triage, in a worker of its own; a head whose diff is the one -// the newest review read takes that review over instead (code-review-runs.ts). +// Reads a pull request as its host shows it, alone or with its diff, and builds the code review request a manager's +// review reads: the diff cut to the limit and hashed whole, so a later head with the same diff can take a review over +// (code-review-runs.ts). import { createHash } from "node:crypto"; import * as bitbucket from "@corbits/bitbucket-datacenter-reads"; import { readPr, readPrDiff, type GithubFetch } from "@corbits/github-reads"; import { shownBody, visibleText, type RepoSource } from "@corbits/triage-contracts"; import type { TenantBitbucket } from "./bitbucket-datacenter/tenant-bitbucket.js"; import { filesFrom } from "@corbits/common/diff-files"; -import type { TriageJob } from "./github/coordinator.js"; -import type { PrMail } from "./github/normalize.js"; import { forInstallation } from "./github/open-pulls.js"; -import { CODE_REVIEW_WORKFLOWS, codeReviewWorkflowFor } from "./github/source-workflow.js"; import { sourceOf, type RepoRecord } from "./github/tenant-config.js"; -const REVIEW_ACTIONS = new Set(["opened", "reopened", "synchronize", "ready_for_review"]); - /** * The most diff text one review reads, about 25k tokens, which keeps a review near its measured cost; past it the diff * is cut at a line boundary and the review names the files it left unread. */ const DIFF_LIMIT = 100_000; -/** Heads remembered as sent, so a webhook delivered twice is reviewed once. */ -const REMEMBERED_HEADS = 10_000; - /** `labels` only where the host has them. */ export type PullRequestHead = { title: string; description: string; headSha: string; draft: boolean; open: boolean; labels?: string[] }; export type PullRequestToReview = PullRequestHead & { diff: string }; -export type CodeReviewFeedDeps = { - /** - * The hosts whose code review runs for `record`: a repository of the host has it on and a code review provider's key - * is stored; none when the PR workflow reviews the repository, so a head is never reviewed twice. - */ - reviewed: (tenantId: string, record: RepoRecord) => Promise; - read: (tenantId: string, record: RepoRecord, number: number) => Promise; - /** Whether the head takes the pull request's newest review over, its diff being the one that review read. */ - carries: (tenantId: string, record: RepoRecord, number: number, pull: PullRequestToReview) => Promise; - send: (job: TriageJob) => Promise; - log: (entry: Record) => void; -}; - -type Unreviewed = "not_reviewed" | "draft" | "not_open"; - /** * The diff's hash, its `index` lines' blob ids included, so a base change to a file the diff touches changes it; undefined * for a diff with no `index` line, such as one rebuilt from the file list, which can't show such a change. @@ -150,79 +127,3 @@ export function reviewRequestOf(record: Pick, num ...(cut && { cut }), }; } - -function isReviewEvent(mail: PrMail): boolean { - return mail.event === "pull_request" && mail.action !== null && REVIEW_ACTIONS.has(mail.action) && mail.draft !== true; -} - -export function createCodeReviewFeed(deps: CodeReviewFeedDeps) { - /** Heads sent or being sent; a head is reserved before its send, so two deliveries of one push send it once. */ - const sent = new Set(); - - function keyOf(tenantId: string, record: Pick, number: number, headSha: string): string { - return `${tenantId}\n${sourceOf(record)}\n${record.name}\n${number}\n${headSha}`; - } - - function reserve(key: string): boolean { - if (sent.has(key)) return false; - sent.add(key); - const [oldest] = sent; - if (sent.size > REMEMBERED_HEADS && oldest !== undefined) sent.delete(oldest); - return true; - } - - /** The pull request to review and its request, or why it is not reviewed. */ - async function toReview(tenantId: string, record: RepoRecord, number: number): Promise<{ pull: PullRequestToReview; request: ReviewRequest } | Unreviewed> { - if (record.codeReview !== true || !(await deps.reviewed(tenantId, record)).includes(sourceOf(record))) return "not_reviewed"; - const pull = await deps.read(tenantId, record, number); - if (!pull.open) return "not_open"; - if (pull.draft) return "draft"; - return { pull, request: reviewRequestOf(record, number, pull) }; - } - - /** Sends the reserved head, which is let go when the send fails. */ - async function send(tenantId: string, record: RepoRecord, number: number, request: ReviewRequest): Promise { - const headSha = request.checkout.sha; - try { - await deps.send({ tenantId, workflow: codeReviewWorkflowFor(sourceOf(record)).id, repo: record.name, number, headSha, source: "event", payload: request }); - } catch (err) { - sent.delete(keyOf(tenantId, record, number, headSha)); - throw err; - } - deps.log({ level: "info", msg: "code_review_sent", tenantId, repo: record.name, pr: number, headSha, ...(request.cut && { cut: request.cut }) }); - } - - /** A new head is sent once, unless it takes the newest review over. */ - async function review(tenantId: string, record: RepoRecord, number: number): Promise { - const found = await toReview(tenantId, record, number); - if (typeof found === "string") return; - const { pull, request } = found; - if (!reserve(keyOf(tenantId, record, number, pull.headSha))) return; - if (await deps.carries(tenantId, record, number, pull)) { - deps.log({ level: "info", msg: "code_review_carried", tenantId, repo: record.name, pr: number, headSha: pull.headSha }); - return; - } - await send(tenantId, record, number, request); - } - - return { - /** A webhook's pull request event; never throws, as the webhook's own handling must not depend on it. */ - async offer(tenantId: string, record: RepoRecord, mail: PrMail): Promise { - if (mail.prNumber === null || !isReviewEvent(mail)) return; - // Checked before reading the host, so a webhook delivered twice costs no second read. - if (mail.headSha !== null && sent.has(keyOf(tenantId, record, mail.prNumber, mail.headSha))) return; - try { - await review(tenantId, record, mail.prNumber); - } catch (err) { - deps.log({ level: "error", msg: "code_review_send_failed", tenantId, repo: record.name, pr: mail.prNumber, error: String(err) }); - } - }, - /** A queued review whose worker could not run, so a later event for its head sends it again. */ - forget(job: TriageJob): void { - const host = CODE_REVIEW_WORKFLOWS.find((workflow) => workflow.id === job.workflow)?.source; - if (host) sent.delete(keyOf(job.tenantId, { name: job.repo, source: host }, job.number, job.headSha)); - }, - }; -} - -export type CodeReviewFeed = ReturnType; diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index f72a8a899..155c2a61c 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -52,14 +52,14 @@ import { authMethods } from "./auth.js"; import { createIntegrationsApp, INTEGRATIONS_PREFIX, logRefusal } from "./integrations/app.js"; import { createBitbucketCodeReviewSend } from "./bitbucket-datacenter/code-review-send.js"; import { createBitbucketIntegration } from "./bitbucket-datacenter/integration.js"; -import { checkedBitbucketSpawner, createTenantBitbucket } from "./bitbucket-datacenter/tenant-bitbucket.js"; -import { createPauseGate, PR_WORKFLOW_VARIANTS, prWorkflowVariantOf } from "./github/pr-workflow.js"; -import { codeReviewSources, codeReviewWorkflowFor, connectedSources, createSourceDeployments, jobWorkflowsFor, TRIAGE_WORKFLOWS, triageWorkflowFor, triageWorkflowOf, type TriageWorkflow } from "./github/source-workflow.js"; +import { checkedBitbucketDelivery, createTenantBitbucket } from "./bitbucket-datacenter/tenant-bitbucket.js"; +import { createPauseGate, PR_WORKFLOW_VARIANTS } from "./github/pr-workflow.js"; +import { codeReviewWorkflowFor, connectedSources, createHostRelease, TRIAGE_WORKFLOWS, triageWorkflowFor, type TriageWorkflow } from "./github/source-workflow.js"; import { createInvitations } from "./invitations/invitations.js"; import { createSmtpEmail } from "./invitations/invite-email.js"; import { createWorkflowDeploy } from "./workflow-deploy.js"; import { createCodeReviewPlace, createCodeReviewPlacement, createCodeReviewRun, createCodeReviewSwitch } from "./code-review.js"; -import { createCodeReviewFeed, createPullRequestReader } from "./code-review-feed.js"; +import { createPullRequestReader } from "./pull-request-reader.js"; import { createCodeReviewRuns } from "./code-review-runs.js"; import { createWorkflowVersions } from "./workflow-versions.js"; import { bitbucketEgressAllowlist, childProcessEnv, clientIpHeaders, databaseConfig, dockerSettings, e2bSettings, emailSettings, interchangeSettings, githubApiOrigin, loadHubEnv, inviteLinkOrigin, railwaySandboxSettings, signInSettings, triageCoalesceWindow, triageReconcileIntervalMs, triageReconcilePolicy } from "./env.js"; @@ -73,10 +73,8 @@ import { createProviderHookApp, isProviderHookPath } from "./provider-hooks/app. import { createProviderHookHandler } from "./provider-hooks/intake.js"; import { MAX_BODY_BYTES } from "./read-json.js"; import { createCoalescer, type Coalescer } from "./github/coalescer.js"; -import { createCoordinator, type Coordinator, type TriageJob } from "./github/coordinator.js"; -import { createWorkerSpawner, releaseOrphanedWorkers } from "./github/worker-spawn.js"; import { DeliveryCache } from "./github/dedupe.js"; -import { activeTenants, createDeploymentRotation, readyMaterializer, recentWorkers, systemSenderMaterializer, resolveLiveDeployments, TriageStoppedError, withWorkflowVersions, type LiveDeployment } from "./github/deployment.js"; +import { activeTenants, readyMaterializer, recentWorkers, systemSenderMaterializer, resolveLiveDeployments, sameSource, TriageStoppedError, withWorkflowVersions, type LiveDeployment } from "./github/deployment.js"; import { createGithubOpenPulls, createMemberOpenPulls, createOpenPullsCache, keptOpenPrs } from "./github/open-pulls.js"; import { createRepoReads } from "./github/repo-reads.js"; import { createGithubPrActions } from "./github/pr-actions.js"; @@ -108,7 +106,7 @@ import { createRepoActivity } from "./github/repo-activity.js"; import { createTenantGithub, createTenantHeadChecks, createTenantOpenHead, createTenantOpenHeads, createTenantOpenPrs } from "./github/tenant-open-heads.js"; import { createSettledStatusReader, createTriageRuns } from "./github/triage-runs.js"; import { createTriageStateStore } from "./github/triage-state-store.js"; -import { failHead, markBaseMoved } from "./github/triage-queue.js"; +import { markBaseMoved, type TriageJob } from "./github/triage-queue.js"; import { createGithubManifestIntegration } from "./github/manifest.js"; import { prepareCorbitsTriagePatch, repoRecords, sourceOf, tenantManager, triageNs } from "./github/tenant-config.js"; import { workflow as prTriageGithub } from "@corbits/triage-workflows/pr-triage-github"; @@ -152,10 +150,7 @@ const TRIAGE_RELIST_QUIET_MS = 30 * 60_000; const TRIAGE_RELIST_GRACE_MS = 60_000; /** How often the live logs of tenants with a pr-triage run in flight are read and recorded, and how stale a shared read may be. */ const TRIAGE_RECORD_MS = 2_000; -const WORKER_FIRE_TIMEOUT_MS = 120_000; -const WORKER_RUN_TIMEOUT_MS = 60 * 60_000; const GITHUB_CALLS_LOG_MS = 60_000; -const WORKER_POLL_MS = 2_000; /** A tenant whose logs could not be read is left at most this long before the recording loop tries again. */ const TRIAGE_RECORD_MAX_BACKOFF_MS = 5 * 60_000; /** How often E2B sandboxes are swept and extended; well inside their one-hour timeout. */ @@ -210,7 +205,6 @@ let triageLoop: ReturnType | undefined; let recordLoop: ReturnType | undefined; let tokenLoop: ReturnType | undefined; let coalescer: Coalescer | undefined; -let coordinator: Coordinator | undefined; let e2bTender: ReturnType | undefined; function logGithubCalls(): void { const line = githubUsage.flush(); @@ -225,7 +219,6 @@ async function shutdown(): Promise { recordLoop?.stop(); tokenLoop?.stop(); coalescer?.stop(); - coordinator?.stop(); clearInterval(e2bTender); clearInterval(githubCallsLogger); try { @@ -337,7 +330,9 @@ const toListener = createManagerIntake({ gate: function gateOf(tenantId, record, number, request) { return pauseGate(tenantId, record, number, request); }, - deliver: deliverToDeployment, + deliver: function deliverOf(tenantId, address, payload) { + return deliverToListener(tenantId, address, payload); + }, log: logJson, }); mountTenantRoutes(composition.app, { db: composition.db, requireGrant, log: logJson, now }); @@ -446,9 +441,9 @@ const mergeStates = createMergeStates({ const GITHUB_SENDER = TRIAGE_MAIL_SENDER; const githubDeliverer = runTriggerDeliverer(GITHUB_SENDER); -async function deliverToDeployment(tenantId: string, address: string, payload: unknown): Promise { +const deliverToListener = checkedBitbucketDelivery(async function deliverMail(tenantId, address, payload) { await githubDeliverer.to(address, JSON.stringify(await withAppLogin(composition.db, tenantId, payload)), tenantId, undefined); -} +}, tenantBitbucket); const githubToken = createGithubTokenRenewal({ ...credentialDeps, githubApiOrigin: githubOrigin, router: composition.sidecarRouter, loadRunCredentials: composition.loadRunCredentials, log: logJson }); for (const tenantId of movedAppKeys) await githubToken.push(tenantId); tokenLoop = createReconcileLoop(githubToken.renewAll, { intervalMs: GITHUB_TOKEN_RENEW_MS, retryMs: GITHUB_TOKEN_RETRY_MS, retryMaxMs: GITHUB_TOKEN_RETRY_MAX_MS }, logJson); @@ -460,20 +455,6 @@ async function renewGithubToken(tenantId: string): Promise { logJson({ level: "error", msg: "github_token_renew_failed", tenantId, error: String(err) }); } } -const GITHUB_JOB_WORKFLOWS = new Set(jobWorkflowsFor("github")); -/** A token that could not be renewed may still have time left, so the worker goes ahead and the token loop retries. */ -async function readyWorkerCredentials(job: TriageJob): Promise { - if (!GITHUB_JOB_WORKFLOWS.has(job.workflow)) return; - try { - await githubToken.ensure(job.tenantId); - } catch (err) { - logJson({ level: "error", msg: "github_token_renew_failed", tenantId: job.tenantId, error: String(err) }); - } -} -/** Every triage and code review mail, whatever started it, runs in a worker of its own. */ -async function sendToWorker(job: TriageJob): Promise { - coordinator!.enqueue(job); -} async function sendTriageMail(job: TriageJob): Promise { await sendToListener(job.tenantId, job.payload); } @@ -489,32 +470,6 @@ const placeCodeReview = createCodeReviewPlacement(composition.db, composition.si async function codeReviewPlaced(tenantId: string): Promise { return (await placeCodeReview(tenantId)) !== null; } -/** Whether the repository's policy file needs the PR workflow, which reviews the code itself. */ -async function inPrWorkflow(tenantId: string, repo: string, source: RepoSource): Promise { - const read = await readCheckPack(tenantId, repo, source); - return read.status === "ok" && read.policyWorkflow !== undefined; -} -// Code review runs in the managers a listener deploys, and nothing runs without one, so the feed sends nothing. -async function reviewedSources(): Promise { - return []; -} -/** The hosts whose PR workflow deploys: a connected repository's policy file needs it, and code review has a place to run. */ -async function prWorkflowSources(tenantId: string): Promise { - if (!(await codeReviewPlaced(tenantId))) return []; - const tenant = await composition.db.query.tenant.findFirst({ where: eq(schema.tenant.id, tenantId), columns: { config: true } }); - const sources = new Set(); - for (const record of repoRecords(triageNs(tenant?.config)).filter((row) => row.connected)) { - if (await inPrWorkflow(tenantId, record.name, sourceOf(record))) sources.add(sourceOf(record)); - } - return [...sources]; -} -const codeReviewFeed = createCodeReviewFeed({ - reviewed: reviewedSources, - read: pullRequests.withDiff, - carries: codeReviewRuns.carries, - send: sendToWorker, - log: logJson, -}); const pauseGate = createPauseGate({ readHead: pullRequests.head, log: logJson }); const tenantSources = { ...credentialDeps, githubApiOrigin: githubOrigin, tenantBitbucket }; const tenantOpenPrs = keptOpenPrs(createTenantOpenPrs(tenantSources), repoReads); @@ -546,6 +501,21 @@ function chosenFileRefusal(tenantId: string, repo: string, source: RepoSource, p async function readCheckPack(tenantId: string, repo: string, source: RepoSource) { return (await settleRepoPolicy(tenantId, repo, source)).read; } +/** Whether the repository's policy file decides code review. */ +async function inPrWorkflow(tenantId: string, repo: string, source: RepoSource): Promise { + const read = await readCheckPack(tenantId, repo, source); + return read.status === "ok" && read.policyWorkflow !== undefined; +} +/** The hosts with a connected repository whose policy file decides code review, while code review has a place to run. */ +async function prWorkflowSources(tenantId: string): Promise { + if (!(await codeReviewPlaced(tenantId))) return []; + const tenant = await composition.db.query.tenant.findFirst({ where: eq(schema.tenant.id, tenantId), columns: { config: true } }); + const sources = new Set(); + for (const record of repoRecords(triageNs(tenant?.config)).filter((row) => row.connected)) { + if (await inPrWorkflow(tenantId, record.name, sourceOf(record))) sources.add(sourceOf(record)); + } + return [...sources]; +} /** A push that lands a file matching the saved rules retires them now, so an edit after it is not taken for a difference. */ async function refreshPolicyFile(tenantId: string, repo: string, source: RepoSource) { const state = await policyFile.onPush(tenantId, repo, source); @@ -624,20 +594,10 @@ function recordingStack(workflowId: string, workerIds: readonly string[] = [work return { observer, recordPass, liveDeployments }; } -/** One host's observer, run recorder, rotation and reconciler over its own workflow's deployments. */ +/** One host's observer, run recorder and reconciler. */ function triageStack(triage: TriageWorkflow) { const variants = PR_WORKFLOW_VARIANTS.filter((variant) => variant.source === triage.source).map((variant) => variant.id); const { observer, recordPass } = recordingStack(triage.id, [triage.id, ...variants]); - const rotation = createDeploymentRotation({ - db: composition.db, - launchSpecs, - allocation: composition.workflowAllocationService, - lifecycle: composition.workflowLifecycleService, - mayDeploy: function mayDeploy(principalId, tenantId) { - return authorizePortal(principalId, tenantId, "workflow:*", "create"); - }, - rotationKey: triage.rotationKey, - }); const reconcile = createTriageReconciler({ source: triage.source, tenants: reconcileTenants, @@ -655,16 +615,14 @@ function triageStack(triage: TriageWorkflow) { now, log: logJson, }); - return { triage, observer, recordPass, rotation, reconcile }; + return { triage, observer, recordPass, reconcile }; } const stacks = TRIAGE_WORKFLOWS.map(triageStack); // pr-triage-historical is only recorded: the reconcilers never mail or rotate it. // A PR workflow variant's listener runs only the approvals of Dos its workers held; those workers are its host's stack's. const recordings = [...stacks, recordingStack(prTriageHistoricalWorkflow.id), ...PR_WORKFLOW_VARIANTS.map((variant) => recordingStack(variant.id, []))]; -// The portal's verdict listing reads every host's observations; connection changes use the GitHub stack's rotation to release. -const github = stacks.find((stack) => stack.triage.source === "github")!; -const deploymentRotation = github.rotation; +// The portal's verdict listing reads every host's observations. const tenantObserver = { last(tenantId: string) { const observations = recordings.flatMap((stack) => stack.observer.last(tenantId) ?? []); @@ -684,6 +642,8 @@ async function recordAll() { return { retry: false }; } async function reconcileAll(): Promise { + // A host removed through the stock credential route tells the hub nothing, so each pass releases what it left running. + for (const tenant of await reconcileTenants()) await releaseHostsOf(tenant.id); const outcomes: ReconcileOutcome[] = []; for (const stack of stacks) outcomes.push(await stack.reconcile()); const retry = outcomes.some((outcome) => outcome.retry); @@ -720,27 +680,28 @@ const githubHeldDos = createGithubHeldDos({ }, log: logJson, }); -const reconcileSourceDeployments = createSourceDeployments({ +const kickTriage = triageLoop.kick; +const releaseDisconnected = createHostRelease({ connectedSources: function connectedSourcesOf(tenantId) { return connectedSources(composition.db, tenantId); }, - codeReviewSources: function codeReviewSourcesOf(tenantId) { - return codeReviewSources(composition.db, tenantId); - }, - codeReviewPlaced, liveDeployments: function liveOf(tenantId, workflow) { return resolveLiveDeployments(composition.db, tenantId, workflow); }, - release: deploymentRotation.release, + async cancel(tenantId, runId) { + await composition.workflowLifecycleService.requestCancellation(tenantId, runId, "source_disconnected"); + }, log: logJson, }); -const kickTriage = triageLoop.kick; -async function onSourcesChanged(tenantId: string): Promise { +async function releaseHostsOf(tenantId: string): Promise { try { - await reconcileSourceDeployments(tenantId); + await releaseDisconnected(tenantId); } catch (err) { - logJson({ level: "error", msg: "triage_source_deployments_failed", tenantId, error: String(err) }); + logJson({ level: "error", msg: "triage_source_release_failed", tenantId, error: String(err) }); } +} +async function onSourcesChanged(tenantId: string): Promise { + await releaseHostsOf(tenantId); kickTriage(); } const syncInstallations = createInstallationSync({ @@ -758,34 +719,6 @@ function setTimer(fire: () => Promise, ms: number): () => void { clearTimeout(handle); }; } -/** A code review job keeps no state to fail: its head is forgotten, so the next event for it sends it again. */ -async function failCoordinatedJob(job: TriageJob, error: string): Promise { - if (!triageWorkflowOf(job.workflow) && !prWorkflowVariantOf(job.workflow)) { - codeReviewFeed.forget(job); - logJson({ level: "error", msg: "code_review_failed", tenantId: job.tenantId, repo: job.repo, pr: job.number, headSha: job.headSha, error }); - return; - } - await failHead(triageStateStore, job, error, now()); -} -await releaseOrphanedWorkers(composition.db, composition.workflowLifecycleService, logJson); -coordinator = createCoordinator({ - spawner: checkedBitbucketSpawner(createWorkerSpawner({ - db: composition.db, - beforeDeploy: readyWorkerCredentials, - redeploy: github.rotation.redeploy, - deliver: deliverToDeployment, - lifecycle: composition.workflowLifecycleService, - sleep: Bun.sleep, - fireTimeoutMs: WORKER_FIRE_TIMEOUT_MS, - pollMs: WORKER_POLL_MS, - }), tenantBitbucket), - retry: { minMs: TRIAGE_RETRY_MS, maxMs: TRIAGE_RETRY_MAX_MS }, - failed: failCoordinatedJob, - runTimeoutMs: WORKER_RUN_TIMEOUT_MS, - setTimer, - now, - log: logJson, -}); coalescer = createCoalescer({ store: triageStateStore, sendMail: sendTriageMail, @@ -811,7 +744,6 @@ const bridge = createBridgeHandler({ cipher: credentialDeps.cipher, cache: deliveryCache, coalescer, - codeReview: codeReviewFeed, readCheckPack, openHeadFor: tenantOpenHead, recheckOpenPulls, @@ -825,7 +757,6 @@ const providerHooks = createProviderHookApp(createProviderHookHandler({ cipher: credentialDeps.cipher, cache: deliveryCache, coalescer, - codeReview: codeReviewFeed, readCheckPack, recheckOpenPulls, refreshPolicyFile, @@ -929,7 +860,9 @@ const integrations = createIntegrationsApp({ db: composition.db, getSession: com artifacts: composition.db, readCheckPack: readStoredCheckPack, runReader: composition.runReader, - sameSource: deploymentRotation.sameSource, + sameSource: function sameSourceOf(anchorRunId, otherAnchorRunId) { + return sameSource(launchSpecs, anchorRunId, otherAnchorRunId); + }, authorize: authorizePortal, workflows: [...TRIAGE_WORKFLOWS, ...PR_WORKFLOW_VARIANTS].map((workflow) => workflow.id), }), diff --git a/apps/hub/src/workflow-versions.test.ts b/apps/hub/src/workflow-versions.test.ts index ec6aed6a8..4091004b4 100644 --- a/apps/hub/src/workflow-versions.test.ts +++ b/apps/hub/src/workflow-versions.test.ts @@ -61,9 +61,9 @@ describe("createWorkflowVersions", () => { expect(res.status).toBe(200); expect(await res.json()).toEqual({ deployments: [ - { id: "run_tarball", workflow: "pr-triage", createdAt: CREATED_AT.toISOString(), cancelling: false, live: true, worker: false, package: WORKFLOW, tools: [TOOL] }, - { id: "run_source", workflow: "pr-triage-historical", createdAt: CREATED_AT.toISOString(), cancelling: true, live: false, worker: false, package: SOURCE_WORKFLOW, tools: [] }, - { id: "run_tarball_worker", workflow: "pr-triage", createdAt: CREATED_AT.toISOString(), cancelling: false, live: true, worker: true, package: WORKFLOW, tools: [TOOL] }, + { id: "run_tarball", workflow: "pr-triage", createdAt: CREATED_AT.toISOString(), cancelling: false, live: true, worker: false, package: WORKFLOW, tools: [TOOL], source: null }, + { id: "run_source", workflow: "pr-triage-historical", createdAt: CREATED_AT.toISOString(), cancelling: true, live: false, worker: false, package: SOURCE_WORKFLOW, tools: [], source: { assetId: "ast_triage", commitSha: "c0ffee" } }, + { id: "run_tarball_worker", workflow: "pr-triage", createdAt: CREATED_AT.toISOString(), cancelling: false, live: true, worker: true, package: WORKFLOW, tools: [TOOL], source: null }, ], }); }); diff --git a/apps/hub/src/workflow-versions.ts b/apps/hub/src/workflow-versions.ts index d65b780b0..babb06454 100644 --- a/apps/hub/src/workflow-versions.ts +++ b/apps/hub/src/workflow-versions.ts @@ -1,6 +1,7 @@ // No stock route exposes a deployment's pin, so the portal reads the versions -// it runs from each live deployment's frozen closure here. +// and source it runs from each live deployment's frozen bundle here. import type { DB, WorkflowRunLaunchSpecStore } from "@intx/db"; +import type { FrozenApprovalBundle } from "@intx/types/sidecar"; import type { DeployedVersions } from "@corbits/triage-contracts"; import { closureVersions, resolveLiveDeployments } from "./github/deployment.js"; import type { Member } from "./integrations/app.js"; @@ -14,17 +15,18 @@ export type WorkflowVersionsDeps = { up: (anchorRunId: string) => Promise; }; -export function createWorkflowVersions(deps: WorkflowVersionsDeps) { - async function versionsOf(anchorRunId: string) { - const spec = await deps.launchSpecs.get(anchorRunId); - return spec ? closureVersions(spec.frozenApprovalBundle) : null; - } +function sourceOf(bundle: FrozenApprovalBundle): DeployedVersions["deployments"][number]["source"] { + if (bundle.source.kind !== "asset" || bundle.source.package.format !== "source") return null; + return { assetId: bundle.source.assetId, commitSha: bundle.source.package.commitSha }; +} +export function createWorkflowVersions(deps: WorkflowVersionsDeps) { return async function handle({ tenantId }: Member): Promise { const deployments: DeployedVersions["deployments"] = []; for (const kind of ["listener", "worker"] as const) { for (const live of await resolveLiveDeployments(deps.db, tenantId, undefined, kind)) { - const versions = await versionsOf(live.runId); + const spec = await deps.launchSpecs.get(live.runId); + const versions = spec ? closureVersions(spec.frozenApprovalBundle) : null; deployments.push({ id: live.runId, workflow: live.workflow, @@ -34,6 +36,7 @@ export function createWorkflowVersions(deps: WorkflowVersionsDeps) { worker: kind === "worker", package: versions?.workflow ?? null, tools: versions?.tools ?? [], + source: spec ? sourceOf(spec.frozenApprovalBundle) : null, }); } } diff --git a/apps/web/src/components/repo-panel/CodeReviewCard.tsx b/apps/web/src/components/repo-panel/CodeReviewCard.tsx index 01a396b85..f9ec7df16 100644 --- a/apps/web/src/components/repo-panel/CodeReviewCard.tsx +++ b/apps/web/src/components/repo-panel/CodeReviewCard.tsx @@ -1,99 +1,23 @@ -import { useId, useState } from "react"; -import { Button, Switch } from "@corbits/react-ui"; -import { useMutation } from "@tanstack/react-query"; +import { Button } from "@corbits/react-ui"; import { Link } from "react-router-dom"; -import { errorText } from "@corbits/common"; -import { useFocusWhenEnabled } from "../../lib/focus-when-enabled.ts"; -import { codeReviewProvider, type RepoRecord } from "../../lib/hub-api.ts"; import { usePortal } from "../../lib/portal.tsx"; -import { KeyPopup } from "../KeyPopup.tsx"; -import { ANTHROPIC_KEY_SAVE, AnthropicKeyForm } from "../settings/AnthropicKeyForm.tsx"; - -const SENTENCE = "Review the code in every pull request for bugs"; - -type NeedsPlaceProps = { switchedOn: boolean; busy: boolean; turnOff: () => void }; - -function NeedsPlace({ switchedOn, busy, turnOff }: NeedsPlaceProps) { - return ( -
-
- Code review needs an isolated place to run - {switchedOn ? "It is switched on here but not running. " : ""}Each review runs apart from everything else, and this server has no such place set up yet. -
-
- - {switchedOn ? : null} -
-
- ); -} - -export default function CodeReviewCard({ repo, disabled }: { repo: RepoRecord; disabled: boolean }) { - const { snapshot, switchCodeReview } = usePortal(); - const [askingForKey, setAskingForKey] = useState(false); - const switchId = useId(); - const keyStored = snapshot !== null && codeReviewProvider(snapshot.credentials) !== undefined; - const keysDenied = !keyStored && snapshot?.denied.credentials === true; - const keysUnreadable = keyStored ? undefined : snapshot?.credentialsError; - const { focusSoon, afterRender } = useFocusWhenEnabled(switchId); - // A key stored elsewhere closes the popup; forgetting the ask keeps it from coming back if that key is removed later. - if (askingForKey && keyStored) setAskingForKey(false); - const change = useMutation({ - mutationFn: function applySwitch(next: boolean) { - return switchCodeReview(repo, next); - }, - }); - const on = change.isPending ? change.variables : repo.codeReview === true; - - function flip(next: boolean) { - if (next && !keyStored) setAskingForKey(true); - else change.mutate(next); - } - - function stopAsking() { - setAskingForKey(false); - } - - function turnOnWithKey() { - setAskingForKey(false); - change.mutate(true); - } - - if (snapshot?.codeReviewPlace === undefined) return null; - if (snapshot.codeReviewPlace === null) { - return ( -
-

Code review

- change.mutate(false)} /> - {change.error ?

Could not change code review. {errorText(change.error)}

: null} -
- ); - } +/** Code review is set in the repository's policy file; this card only says when the server has nowhere to run it. */ +export default function CodeReviewCard() { + const { snapshot } = usePortal(); + if (snapshot?.codeReviewPlace !== null) return null; return (

Code review

-
-
-
- {SENTENCE} - -
+
+
+ Code review needs an isolated place to run + Each review runs apart from everything else, and this server has no such place set up yet. +
+
+
-
- - - - {keysDenied ?

Code review needs an Anthropic key, and only a workspace owner can add one. Ask an owner to add it in Settings → Code review.

: null} - {keysUnreadable === undefined ? null :

Could not read the workspace's keys. {keysUnreadable}

} - {change.error ?

Could not change code review. {errorText(change.error)}

: null} +
); } diff --git a/apps/web/src/components/repo-panel/RepoPanel.tsx b/apps/web/src/components/repo-panel/RepoPanel.tsx index 1a57f5896..83907af3a 100644 --- a/apps/web/src/components/repo-panel/RepoPanel.tsx +++ b/apps/web/src/components/repo-panel/RepoPanel.tsx @@ -69,7 +69,7 @@ export default function RepoPanel({ repo, row, live, store, onClose }: RepoPanel {manualWebhook ? : null}
- + {stats ? : null} diff --git a/apps/web/src/components/settings/RestartFailed.tsx b/apps/web/src/components/settings/RestartFailed.tsx index 741788e00..01b097f9b 100644 --- a/apps/web/src/components/settings/RestartFailed.tsx +++ b/apps/web/src/components/settings/RestartFailed.tsx @@ -2,16 +2,19 @@ import { Button } from "@corbits/react-ui"; import { useMutation, type UseMutationResult } from "@tanstack/react-query"; import { errorText } from "@corbits/common"; import type { RepoSource } from "@corbits/triage-contracts"; +import { createHubTransport } from "../../lib/hub-transport.ts"; import { usePortal } from "../../lib/portal.tsx"; +import { restartListener } from "../../lib/triage-upgrade.ts"; type Restart = UseMutationResult; -/** Restarts the host's deployments after one of its credentials is replaced; started once the replace has succeeded, so it never fails the replace. */ +/** Restarts the host's listener after one of its credentials is replaced; started once the replace has succeeded, so it never fails the replace. */ export function useRestartTriage(source: RepoSource): Restart { - const { redeployHost } = usePortal(); + const { snapshot } = usePortal(); return useMutation({ - mutationFn: function restart() { - return redeployHost(source); + async mutationFn() { + if (!snapshot) throw new Error("The service is unavailable. Reload the page and try again."); + await restartListener(createHubTransport(), snapshot.workspace.tenantId, source); }, }); } diff --git a/apps/web/src/lib/hub-api.ts b/apps/web/src/lib/hub-api.ts index 2a5f5fcb9..9113aaaa3 100644 --- a/apps/web/src/lib/hub-api.ts +++ b/apps/web/src/lib/hub-api.ts @@ -136,8 +136,6 @@ export type PortalSnapshot = { /** Absent when the hub refused or failed to say where code review runs. */ codeReviewPlace?: CodeReviewPlace["provisioner"]; codeReviewPlaces?: CodeReviewPlace["places"]; - /** The hosts whose PR workflow deploys; absent with `codeReviewPlace`. */ - prWorkflowSources?: CodeReviewPlace["prWorkflow"]; /** Set when the hub failed to list the keys for a reason other than access, so an empty `credentials` does not mean no key is stored. */ credentialsError?: string; /** Set when the hub failed to say where code review runs for a reason other than access. */ @@ -903,21 +901,11 @@ export async function reviewAgain(transport: Transport, tenantId: string, repo: await transport.fetch("POST", `/api/integrations/code-review/${enc(requireTenantId(tenantId))}/run`, { repo: validateRepo(repo.name), source: repo.source, number }); } -export async function setCodeReview(transport: Transport, tenantId: string, repo: Pick, enabled: boolean): Promise { - await transport.fetch("POST", `/api/integrations/code-review/${enc(requireTenantId(tenantId))}`, { repo: validateRepo(repo.name), source: repo.source, enabled }); -} - /** The provider the workspace runs code review on: the one whose key it stores in Settings → Code review. */ export function codeReviewProvider(credentials: readonly HubCredential[]): CodeReviewProvider | undefined { return codeReviewProviderOf(credentials.filter(isActiveCredential).map((credential) => credential.name)); } -/** The hosts with a connected repository that has code review on, while a code review provider's key is stored and the hub can run it: the hosts whose code review deploys. */ -export function codeReviewSources(snapshot: Pick): RepoSource[] { - if (!codeReviewProvider(snapshot.credentials) || typeof snapshot.codeReviewPlace !== "string") return []; - return [...new Set(snapshot.repos.filter((repo) => repo.connected && repo.codeReview).map((repo) => repo.source))]; -} - export type OpenPulls = { repos: Array<{ repo: string; @@ -1261,7 +1249,7 @@ export async function loadPortal(transport: Transport, access: MemberAccess, has configVersion: configFingerprint(tenant.value.config), credentials: credentials.value, ...(decisionModel && { decisionModel }), - ...(places.value && { codeReviewPlace: places.value.provisioner, codeReviewPlaces: places.value.places, prWorkflowSources: places.value.prWorkflow }), + ...(places.value && { codeReviewPlace: places.value.provisioner, codeReviewPlaces: places.value.places }), ...(credentials.failure && { credentialsError: credentials.failure.message }), ...(places.failure && { codeReviewPlaceError: places.failure.message }), denied: { diff --git a/apps/web/src/lib/portal.tsx b/apps/web/src/lib/portal.tsx index 4ac1ace18..b1e607fb9 100644 --- a/apps/web/src/lib/portal.tsx +++ b/apps/web/src/lib/portal.tsx @@ -16,16 +16,12 @@ import { ensureWorkspace, loadPortal, legacyHookCredentials, - connectedSources, portalConnected, replaceCredential, removeRepository, resolveApproval, revokeCredential, saveAnthropicKey, - setCodeReview, - codeReviewProvider, - codeReviewSources, startBacklogTriage, githubPrAction, triagePullRequest as requestPullRequestTriage, @@ -41,11 +37,10 @@ import { } from "./hub-api.ts"; import { readChosenWorkspace, writeChosenWorkspace } from "./chosen-workspace.ts"; import type { CreateGrantInput } from "./grant-actions.ts"; -import { ensureWorkflows, releaseWorkflows, type DeployHosts, type Redeploy } from "./workflow-deploy.ts"; import { reconcileModelCatalog, removeModelKey, saveInference } from "./model-catalog.ts"; import { CHECK_PACK_INDEX_QUERY_KEY, checkPackIndexQuery } from "./check-packs.ts"; import { syncGithubInstallations, type SyncResult } from "./github-manifest.ts"; -import { checkPackName, type RepoSource } from "@corbits/triage-contracts"; +import { checkPackName } from "@corbits/triage-contracts"; /** `now` confirms the write with a toast; `held` writes were announced by the inbox's Undo toast; `leaving` ones are held writes sent as the page goes away, so they must outlive it. */ type GithubDelivery = "now" | "held" | "leaving"; @@ -74,15 +69,12 @@ interface PortalContextValue { writeGithub: (input: PrGithubWriteInput, delivery: GithubDelivery) => Promise; decide: (approvalId: string, decision: "once" | "always" | "deny") => Promise; replaceSecret: (credentialId: string, secret: string) => Promise; - /** Redeploys a host's listener and PR workflow, which hold its credentials from when they were deployed; rejects when that fails. */ - redeployHost: (source: RepoSource) => Promise; revoke: (credentialId: string) => Promise; addGrant: (input: CreateGrantInput) => Promise; saveInferenceSecret: (input: { endpoint: string; model: string; secret: string }) => Promise; saveAnthropic: (secret: string) => Promise; removeAnthropic: (credentialId: string) => Promise; removeDecisionModel: (credentialId: string) => Promise; - switchCodeReview: (repo: Pick, enabled: boolean) => Promise; } const NO_INVITATIONS: WorkspaceOption[] = []; @@ -165,24 +157,6 @@ export function useSignOutWhenRejected(): (cause: unknown) => void { }, [signOut]); } -/** Each tenant's converge, keyed by the hosts it was connected to, so connecting or disconnecting one converges again. */ -const convergedTenants = new Set(); - -const NO_HOSTS: DeployHosts = { connected: [], codeReview: [], codeReviewProvider: undefined, prWorkflow: undefined }; -const DEPLOY_MISSING: Redeploy = { kinds: new Set() }; -// The PR workflow triages on the decision model too. -const REDEPLOY_TRIAGE: Redeploy = { kinds: new Set(["triage", "pr-workflow"]) }; -const REDEPLOY_CODE_REVIEW: Redeploy = { kinds: new Set(["code-review", "pr-workflow"]) }; - -/** A workspace's converges are remembered per set of hosts; switching code review back on must deploy it again even when that set was seen before. */ -function forgetConverged(tenantId: string): void { - for (const key of convergedTenants) if (key.startsWith(`${tenantId}:`)) convergedTenants.delete(key); -} - -function hostsOf(snapshot: PortalSnapshot): DeployHosts { - return { connected: connectedSources(snapshot.credentials), codeReview: codeReviewSources(snapshot), codeReviewProvider: codeReviewProvider(snapshot.credentials), prWorkflow: snapshot.prWorkflowSources }; -} - type PortalLoad = PortalSnapshot | InvitedAccess; function isInvited(load: PortalLoad): load is InvitedAccess { @@ -271,34 +245,6 @@ export function PortalProvider({ children }: { children: ReactNode }) { } }, [enabled, queryClient]); - const tenantId = snapshot?.workspace.tenantId; - const hosts = useMemo(() => (snapshot ? hostsOf(snapshot) : NO_HOSTS), [snapshot]); - const decisionModelConnected = snapshot?.decisionModel !== undefined; - const deployWorkflows = useCallback(async function deployWorkflows(id: string, deployHosts: DeployHosts, redeploy: Redeploy) { - const transport = createHubTransport(); - await reconcileModelCatalog(transport, id); - const deployed = await ensureWorkflows(transport, id, deployHosts, redeploy); - if (deployed.length) refresh(); - }, [refresh]); - const convergeWorkflows = useCallback(function convergeWorkflows(id: string, deployHosts: DeployHosts, redeploy: Redeploy) { - const key = `${id}:${JSON.stringify(deployHosts)}`; - if (redeploy.kinds.size === 0 && convergedTenants.has(key)) return; - convergedTenants.add(key); - async function deploy() { - try { - await deployWorkflows(id, deployHosts, redeploy); - } catch (cause: unknown) { - convergedTenants.delete(key); - warn(`Could not deploy triage workflows. ${errorText(cause)}`); - } - } - void deploy(); - }, [deployWorkflows, warn]); - - useEffect(function convergeOnConnect() { - if (tenantId && hosts.connected.length > 0 && decisionModelConnected) convergeWorkflows(tenantId, hosts, DEPLOY_MISSING); - }, [convergeWorkflows, decisionModelConnected, hosts, tenantId]); - const requireSnapshot = useCallback(function requireSnapshot() { if (readOnly || !snapshot) { throw new Error("The service is unavailable. Reload the page and try again."); @@ -383,14 +329,6 @@ export function PortalProvider({ children }: { children: ReactNode }) { [notify, refresh, requireSnapshot], ); - const redeployHost = useCallback(async function redeployHost(source: RepoSource) { - const fresh = await refreshNow(); - // Every deploy includes triage, which needs the decision model. - if (!fresh?.decisionModel) return; - const freshHosts = hostsOf(fresh); - if (freshHosts.connected.includes(source)) await deployWorkflows(fresh.workspace.tenantId, freshHosts, { ...REDEPLOY_TRIAGE, sources: [source] }); - }, [deployWorkflows, refreshNow]); - const replaceSecret = useCallback( async function replaceSecret(credentialId: string, secret: string) { const current = requireSnapshot(); @@ -435,12 +373,11 @@ export function PortalProvider({ children }: { children: ReactNode }) { async function saveInferenceSecret(input: { endpoint: string; model: string; secret: string }) { const current = requireSnapshot(); await saveInference(createHubTransport(), current.workspace.tenantId, input); + await reconcileModelCatalog(createHubTransport(), current.workspace.tenantId); notify("Decision model saved."); refresh(); - const deployHosts = hostsOf(current); - if (deployHosts.connected.length > 0) convergeWorkflows(current.workspace.tenantId, deployHosts, REDEPLOY_TRIAGE); }, - [convergeWorkflows, notify, refresh, requireSnapshot], + [notify, refresh, requireSnapshot], ); const saveAnthropic = useCallback( @@ -450,22 +387,15 @@ export function PortalProvider({ children }: { children: ReactNode }) { await saveAnthropicKey(transport, current.workspace.tenantId, secret); await reconcileModelCatalog(transport, current.workspace.tenantId); notify("Anthropic key saved."); - // A new key can mean new offerings, which a deployment resolves only when deployed. - const fresh = await refreshNow(); - // Every converge deploys triage, which needs the decision model. - if (!fresh?.decisionModel) return; - const freshHosts = hostsOf(fresh); - if (freshHosts.codeReview.length > 0 || (freshHosts.prWorkflow ?? []).length > 0) convergeWorkflows(fresh.workspace.tenantId, freshHosts, REDEPLOY_CODE_REVIEW); + refresh(); }, - [convergeWorkflows, notify, refreshNow, requireSnapshot], + [notify, refresh, requireSnapshot], ); const removeAnthropic = useCallback( async function removeAnthropic(credentialId: string) { const current = requireSnapshot(); const transport = createHubTransport(); - await releaseWorkflows(transport, current.workspace.tenantId, "code-review"); - await releaseWorkflows(transport, current.workspace.tenantId, "pr-workflow"); await removeModelKey(transport, current.workspace.tenantId, ANTHROPIC_PROVIDER, credentialId); notify("Anthropic key removed."); refresh(); @@ -477,8 +407,6 @@ export function PortalProvider({ children }: { children: ReactNode }) { async function removeDecisionModel(credentialId: string) { const current = requireSnapshot(); const transport = createHubTransport(); - await releaseWorkflows(transport, current.workspace.tenantId, "triage"); - await releaseWorkflows(transport, current.workspace.tenantId, "pr-workflow"); await removeModelKey(transport, current.workspace.tenantId, INFERENCE_PROVIDER, credentialId); notify("Secret removed."); refresh(); @@ -486,18 +414,6 @@ export function PortalProvider({ children }: { children: ReactNode }) { [notify, refresh, requireSnapshot], ); - const switchCodeReview = useCallback( - async function switchCodeReview(repo: Pick, enabled: boolean) { - const current = requireSnapshot(); - await setCodeReview(createHubTransport(), current.workspace.tenantId, repo, enabled); - notify(enabled ? `Code review is on for ${repo.name}.` : `Code review is off for ${repo.name}.`); - // A host whose code review was off has no deployment, so the converge the fresh snapshot starts must deploy it. - if (enabled && !hostsOf(current).codeReview.includes(repo.source)) forgetConverged(current.workspace.tenantId); - await refreshNow(); - }, - [notify, refreshNow, requireSnapshot], - ); - const value = useMemo( function portalValue() { return { @@ -522,14 +438,12 @@ export function PortalProvider({ children }: { children: ReactNode }) { writeGithub, decide, replaceSecret, - redeployHost, revoke, addGrant, saveInferenceSecret, saveAnthropic, removeAnthropic, removeDecisionModel, - switchCodeReview, }; }, [ @@ -553,14 +467,12 @@ export function PortalProvider({ children }: { children: ReactNode }) { writeGithub, decide, replaceSecret, - redeployHost, revoke, addGrant, saveInferenceSecret, saveAnthropic, removeAnthropic, removeDecisionModel, - switchCodeReview, ], ); diff --git a/apps/web/src/lib/triage-upgrade.test.ts b/apps/web/src/lib/triage-upgrade.test.ts index 531158efa..9a82f0c50 100644 --- a/apps/web/src/lib/triage-upgrade.test.ts +++ b/apps/web/src/lib/triage-upgrade.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import type { Transport } from "@intx/hub-client"; -import { upgradeTriage, type UpgradeStep } from "./triage-upgrade.ts"; +import { restartListener, upgradeTriage, type UpgradeStep } from "./triage-upgrade.ts"; import type { PublishedSource } from "./workflow-sources.ts"; const PUBLISHER = { tenantId: "t", principalId: "prn_deployer" }; @@ -9,17 +9,29 @@ const PUBLISHED: PublishedSource[] = [ { workflow: "triage-listener-github", assetId: "ast_listener", commitSha: "l1" }, ]; -/** Live listeners: one the owner published, one another Deployer published, and another workflow's. */ +/** Live listeners one the owner and another Deployer published, the workflows they replace, a manager, and another host's pr-triage. */ function fakeHub() { const deployments = [ { id: "run_owner", tenantId: "t", definitionAssetId: "ast_owner_listener", status: "deployed", createdAt: "2026-10-01T00:00:00.000Z" }, { id: "run_other", tenantId: "t", definitionAssetId: "ast_other_listener", status: "deployed", createdAt: "2026-10-02T00:00:00.000Z" }, { id: "run_triage", tenantId: "t", definitionAssetId: "ast_triage", status: "deployed", createdAt: "2026-10-02T00:00:00.000Z" }, + { id: "run_review", tenantId: "t", definitionAssetId: "ast_review", status: "deployed", createdAt: "2026-10-02T00:00:00.000Z" }, + { id: "run_manager", tenantId: "t", definitionAssetId: "ast_manager", status: "deployed", createdAt: "2026-10-02T00:00:00.000Z" }, + { id: "run_bitbucket", tenantId: "t", definitionAssetId: "ast_bitbucket", status: "deployed", createdAt: "2026-10-02T00:00:00.000Z" }, ]; - const workflows: Record = { run_owner: "triage-listener-github", run_other: "triage-listener-github", run_triage: "pr-triage-github" }; + const workflows: Record = { + run_owner: "triage-listener-github", + run_other: "triage-listener-github", + run_triage: "pr-triage-github", + run_review: "pr-code-review-github", + run_manager: "triage-manager-github", + run_bitbucket: "pr-triage-bitbucket-datacenter", + }; const requests: Array<{ method: string; path: string; body?: unknown }> = []; async function json(method: string, path: string, body?: unknown): Promise { requests.push({ method, path, ...(body !== undefined && { body }) }); + if (method === "GET" && path === "/api/tenants/t") return { config: {} } as T; + if (method === "PATCH" && path === "/api/tenants/t") return undefined as T; if (path === "/api/tenants/t/models") return [{ canonicalName: "decision", offerings: [{ offeringId: "mof_1", providerName: "corbits-system-one" }] }] as T; if (method === "POST" && path === "/api/tenants/t/workflows/deployments") { const created = { id: "run_new", tenantId: "t", definitionAssetId: "ast_listener", status: "deployed", createdAt: "2026-10-03T00:00:00.000Z" }; @@ -29,7 +41,7 @@ function fakeHub() { } if (path === "/api/tenants/t/workflows/deployments") return deployments as T; if (path === "/api/integrations/workflow-versions/t") { - return { deployments: deployments.filter((row) => row.status === "deployed").map((row) => ({ id: row.id, workflow: workflows[row.id], createdAt: row.createdAt, cancelling: false, live: true, worker: false, package: null, tools: [] })) } as T; + return { deployments: deployments.filter((row) => row.status === "deployed").map((row) => ({ id: row.id, workflow: workflows[row.id], createdAt: row.createdAt, cancelling: false, live: true, worker: false, package: null, tools: [], source: { assetId: row.definitionAssetId, commitSha: `c_${row.id}` } })) } as T; } if (method === "DELETE" && path.startsWith("/api/tenants/t/workflows/runs/")) { deployments.find((row) => path.endsWith(row.id))!.status = "cancelled"; @@ -40,12 +52,13 @@ function fakeHub() { return { transport: { fetch: json } as unknown as Transport, requests }; } -test("an upgrade deploys the listener from its published source, then stops every other listener of the host, whoever published it", async () => { +test("an upgrade deploys the listener from its published source, then stops the host's other listeners, whoever published them, and the workflows it replaces", async () => { const hub = fakeHub(); const steps: UpgradeStep[] = []; await upgradeTriage(hub.transport, PUBLISHER, ["github"], (step) => steps.push(step), async () => PUBLISHED); expect(steps).toEqual(["publishing", "starting", "stopping"]); + expect(hub.requests).toContainEqual({ method: "PATCH", path: "/api/tenants/t", body: { config: { lifecycle: { maxLifetime: "36500d" } } } }); expect(hub.requests.find((row) => row.method === "POST")?.body).toEqual({ source: { kind: "asset", assetId: "ast_listener", package: { format: "source", commitSha: "l1", packageName: "@corbits/triage-listener-github-workflow" } }, entry: "./triage-listener-github.mjs", @@ -55,9 +68,24 @@ test("an upgrade deploys the listener from its published source, then stops ever expect(hub.requests.filter((row) => row.method === "DELETE").map((row) => row.path)).toEqual([ "/api/tenants/t/workflows/runs/run_owner", "/api/tenants/t/workflows/runs/run_other", + "/api/tenants/t/workflows/runs/run_triage", + "/api/tenants/t/workflows/runs/run_review", ]); }); +test("restarting a host's listener after a credential is replaced deploys its newest listener again from the same source and stops the older ones", async () => { + const hub = fakeHub(); + await restartListener(hub.transport, "t", "github"); + expect(hub.requests.find((row) => row.method === "POST")?.body).toMatchObject({ + source: { kind: "asset", assetId: "ast_other_listener", package: { format: "source", commitSha: "c_run_other", packageName: "@corbits/triage-listener-github-workflow" } }, + }); + expect(hub.requests.filter((row) => row.method === "DELETE").map((row) => row.path)).toEqual(expect.arrayContaining([ + "/api/tenants/t/workflows/runs/run_owner", + "/api/tenants/t/workflows/runs/run_other", + ])); + expect(hub.requests.some((row) => row.path.endsWith("run_manager") || row.path.endsWith("run_bitbucket"))).toBe(false); +}); + type Row = { id: string; tenantId: string; definitionAssetId: string; status: string; createdAt: string }; function sharedHub() { @@ -66,7 +94,9 @@ function sharedHub() { function transportFor(who: string): Transport { async function json(method: string, path: string): Promise { await Promise.resolve(); - if (path === "/api/tenants/t/models") return [{ canonicalName: "decision", offerings: [{ offeringId: "mof_1", providerName: "corbits-system-one" }] }] as T; + if (method === "GET" && path === "/api/tenants/t") return { config: {} } as T; + if (method === "PATCH" && path === "/api/tenants/t") return undefined as T; + if (path === "/api/tenants/t/models") return [{ canonicalName: "decision", offerings: [{ offeringId: "mof_1", providerName: "corbits-system-one" }] }] as T; if (method === "POST" && path === "/api/tenants/t/workflows/deployments") { const created = { id: `run_${who}`, tenantId: "t", definitionAssetId: `ast_${who}`, status: "deployed", createdAt: `2026-10-03T00:00:0${n++}.000Z` }; deployments.push(created); @@ -74,7 +104,7 @@ function sharedHub() { } if (path === "/api/tenants/t/workflows/deployments") return deployments.map((row) => ({ ...row })) as T; if (path === "/api/integrations/workflow-versions/t") { - return { deployments: deployments.filter((row) => row.status === "deployed").map((row) => ({ id: row.id, workflow: "triage-listener-github", createdAt: row.createdAt, cancelling: false, live: true, worker: false, package: null, tools: [] })) } as T; + return { deployments: deployments.filter((row) => row.status === "deployed").map((row) => ({ id: row.id, workflow: "triage-listener-github", createdAt: row.createdAt, cancelling: false, live: true, worker: false, package: null, tools: [], source: null })) } as T; } if (method === "DELETE" && path.startsWith("/api/tenants/t/workflows/runs/")) { deployments.find((row) => path.endsWith(row.id))!.status = "cancelled"; diff --git a/apps/web/src/lib/triage-upgrade.ts b/apps/web/src/lib/triage-upgrade.ts index aa6dc9693..53b673eb7 100644 --- a/apps/web/src/lib/triage-upgrade.ts +++ b/apps/web/src/lib/triage-upgrade.ts @@ -1,5 +1,6 @@ // Switching a workspace to the per-pull-request manager: publish this portal's release as the person upgrading, deploy -// each connected host's listener from it, and once it is up stop that host's listeners deployed before it. +// each connected host's listener from it, and once it is up stop that host's listeners deployed before it and every +// deployment of the workflows the listener replaced. import { skipToken, useMutation, useQuery } from "@tanstack/react-query"; import { errorText } from "@corbits/common"; import { deployWorkflow, listWorkflowDeployments, type Transport, type WorkflowDeployment } from "@intx/hub-client"; @@ -11,7 +12,7 @@ import { offeringsFor } from "./model-catalog.ts"; import { hostFullName } from "./pull-link.ts"; import { TRIAGE_RELEASE_QUERY_KEY, usePortal } from "./portal.tsx"; import { usePrincipals, useRoles } from "./tenant-entities.ts"; -import { cancelDeployment, portalPackage } from "./workflow-deploy.ts"; +import { cancelDeployment, ensureKeepDeployedLifetime, portalPackage } from "./workflow-deploy.ts"; import { CHAIN_PACKAGES, type SourceIndexEntry } from "./workflow-packages.ts"; import { publishSources, type PublishedSource, type Publisher } from "./workflow-sources.ts"; @@ -25,6 +26,12 @@ const START_DEADLINE_MS = 5 * 60_000; const START_POLL_MS = 2_000; const FAILED_STATUSES = new Set(["failed", "cancelled", "released"]); +/** Each host's workflows from before the listener, which nothing mails any more. */ +const REPLACED: Record = { + github: ["pr-triage", "pr-triage-github", "pr-triage-historical", "pr-code-review-github", "pr-workflow-anthropic-github"], + "bitbucket-datacenter": ["pr-triage-bitbucket-datacenter", "pr-code-review-bitbucket-datacenter", "pr-workflow-anthropic-bitbucket-datacenter"], +}; + type Asset = { id: string; name: string; creatorPrincipalId: string | null }; /** A host's newest live listener: its package version, the release's build hash, and who published it. */ @@ -67,9 +74,14 @@ function olderThan(deployment: { id: string; createdAt: string }, newer: Workflo } /** Only listeners deployed before this one, whoever published them, so of two upgrades at once the newer stays. */ -async function stopOlderListeners(transport: Transport, tenantId: string, source: RepoSource, started: WorkflowDeployment): Promise { +function isOlderListener(deployment: DeployedWorkflow, source: RepoSource, started: WorkflowDeployment): boolean { + return deployment.workflow === TRIAGE_LISTENERS[source] && olderThan(deployment, started) && !deployment.worker; +} + +async function stopOldDeployments(transport: Transport, tenantId: string, source: RepoSource, started: WorkflowDeployment): Promise { for (const deployment of await listDeployedWorkflows(transport, tenantId)) { - if (deployment.workflow === TRIAGE_LISTENERS[source] && olderThan(deployment, started) && !deployment.cancelling && !deployment.worker) await cancelDeployment(transport, tenantId, deployment.id); + if (deployment.cancelling) continue; + if (isOlderListener(deployment, source, started) || REPLACED[source].includes(deployment.workflow)) await cancelDeployment(transport, tenantId, deployment.id); } } @@ -81,21 +93,27 @@ async function newerLive(transport: Transport, tenantId: string, source: RepoSou /** "superseded" when another upgrade's newer listener replaced this one. */ async function upgradeHost(transport: Transport, tenantId: string, source: RepoSource, published: readonly PublishedSource[], onStep: (step: UpgradeStep) => void): Promise<"upgraded" | "superseded"> { onStep("starting"); - const deployment = await deployListener(transport, tenantId, source, published); + return replaceListener(transport, tenantId, source, await deployListener(transport, tenantId, source, published), onStep); +} + +async function replaceListener(transport: Transport, tenantId: string, source: RepoSource, deployment: WorkflowDeployment, onStep: (step: UpgradeStep) => void): Promise<"upgraded" | "superseded"> { if (!(await waitUntilStarted(transport, tenantId, deployment.id))) { if (await newerLive(transport, tenantId, source, deployment)) return "superseded"; throw new Error("The new version did not start, so the current one keeps running."); } onStep("stopping"); - await stopOlderListeners(transport, tenantId, source, deployment); + await stopOldDeployments(transport, tenantId, source, deployment); return (await newerLive(transport, tenantId, source, deployment)) ? "superseded" : "upgraded"; } async function deployListener(transport: Transport, tenantId: string, source: RepoSource, published: readonly PublishedSource[]): Promise { - const workflow = TRIAGE_LISTENERS[source]; - const listener = published.find((row) => row.workflow === workflow); + const listener = published.find((row) => row.workflow === TRIAGE_LISTENERS[source]); if (!listener) throw new Error("Add a decision model before upgrading."); - const chain = chainOf(workflow); + return deployFrom(transport, tenantId, source, listener); +} + +async function deployFrom(transport: Transport, tenantId: string, source: RepoSource, listener: { assetId: string; commitSha: string }): Promise { + const chain = chainOf(TRIAGE_LISTENERS[source]); const offerings = await offeringsFor(transport, tenantId, chain.models); return deployWorkflow(transport, tenantId, { source: { kind: "asset", assetId: listener.assetId, package: { format: "source", commitSha: listener.commitSha, packageName: chain.packageName } }, @@ -105,6 +123,17 @@ async function deployListener(transport: Transport, tenantId: string, source: Re }); } +/** + * A running listener holds the credentials it launched with, so after one is replaced its newest listener is deployed + * again from the same source and the older ones stop; a host with no listener is left for Upgrade. + */ +export async function restartListener(transport: Transport, tenantId: string, source: RepoSource): Promise { + const listeners = (await listDeployedWorkflows(transport, tenantId)).filter((row) => row.workflow === TRIAGE_LISTENERS[source] && !row.cancelling && !row.worker); + const [newest] = listeners.sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id)); + if (!newest?.source) return; + await replaceListener(transport, tenantId, source, await deployFrom(transport, tenantId, source, newest.source), function quiet() {}); +} + /** The hosts where another person's upgrade finished first. */ export async function upgradeTriage( transport: Transport, @@ -114,6 +143,7 @@ export async function upgradeTriage( publish: (transport: Transport, publisher: Publisher) => Promise = publishSources, ): Promise { onStep("publishing"); + await ensureKeepDeployedLifetime(transport, publisher.tenantId); const published = await publish(transport, publisher); const superseded: RepoSource[] = []; for (const source of sources) { diff --git a/apps/web/src/lib/workflow-deploy.test.ts b/apps/web/src/lib/workflow-deploy.test.ts deleted file mode 100644 index 2158e876f..000000000 --- a/apps/web/src/lib/workflow-deploy.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { ApiError, type Transport, type WorkflowDeployment } from "@intx/hub-client"; -import { cancelsSuperseded, ensureWorkflows } from "./workflow-deploy.ts"; -import type { PackageIndexEntry } from "./workflow-packages.ts"; - -const DECISION = [{ canonicalName: "decision", offerings: [{ offeringId: "mof_1", providerName: "corbits-system-one" }] }]; -const BOTH = { connected: ["github", "bitbucket-datacenter"], codeReview: [], codeReviewProvider: "anthropic", prWorkflow: [] } as const; -const GITHUB = { connected: ["github"], codeReview: [], codeReviewProvider: "anthropic", prWorkflow: [] } as const; -const MISSING = { kinds: new Set() }; -const INDEX: PackageIndexEntry[] = [ - { name: "@corbits/github-tool", version: "0.2.0", filename: "@corbits-github-tool-0.2.0.tgz", integrity: "sha512-tool" }, - { name: "@corbits/pr-triage-github-workflow", version: "0.1.0-sha-aaaaaaaa", filename: "@corbits-pr-triage-github-workflow-0.1.0-sha-aaaaaaaa.tgz", integrity: "sha512-triage" }, - { name: "@corbits/pr-triage-bitbucket-datacenter-workflow", version: "0.1.0-sha-cccccccc", filename: "@corbits-pr-triage-bitbucket-datacenter-workflow-0.1.0-sha-cccccccc.tgz", integrity: "sha512-bitbucket" }, - { name: "@corbits/pr-triage-historical-workflow", version: "0.1.0-sha-bbbbbbbb", filename: "@corbits-pr-triage-historical-workflow-0.1.0-sha-bbbbbbbb.tgz", integrity: "sha512-historical" }, - { name: "@corbits/pr-code-review-github-workflow", version: "0.1.0-sha-dddddddd", filename: "@corbits-pr-code-review-github-workflow-0.1.0-sha-dddddddd.tgz", integrity: "sha512-review" }, - { name: "@corbits/pr-workflow-anthropic-github-workflow", version: "0.1.0-sha-eeeeeeee", filename: "@corbits-pr-workflow-anthropic-github-workflow-0.1.0-sha-eeeeeeee.tgz", integrity: "sha512-pr-workflow" }, -]; - -type Asset = { id: string; kind: string; name: string }; - -function packageOf(pin: string) { - const at = pin.lastIndexOf("@"); - return { name: pin.slice(0, at), version: pin.slice(at + 1) }; -} - -function fakeHub(tarballSeed: Record = {}, unconnected: readonly string[] = [], models: unknown[] = DECISION) { - const assets: Asset[] = []; - const tarballs = new Map(Object.entries(tarballSeed)); - const deployments: WorkflowDeployment[] = []; - const pins = new Map(); - const requests: string[] = []; - const deployed: unknown[] = []; - const workers: Array<{ package: { name: string; version: string }; tools: Array<{ name: string; version: string }> }> = []; - - async function json(method: string, path: string, body?: unknown): Promise { - requests.push(`${method} ${path}`); - const url = new URL(path, "http://hub"); - if (path === "/api/tenants/t") return (method === "GET" ? { config: {} } : undefined) as T; - if (path === "/api/tenants/t/workflows/deployments") return deployments as T; - if (path === "/api/tenants/t/models") return models as T; - if (path === "/api/integrations/workflow-versions/t") { - const live = deployments.filter((row) => row.status === "deployed").reverse(); - const listeners = live.map((row) => ({ - id: row.id, - workflow: assets.find((asset) => asset.id === row.definitionAssetId)?.name, - createdAt: row.createdAt, - cancelling: false, - live: true, - worker: false, - package: packageOf(pins.get(row.id)!), - tools: [{ name: INDEX[0]!.name, version: INDEX[0]!.version }], - })); - const running = workers.map((row, n) => ({ ...row, id: `worker_${n}`, workflow: "pr-triage-github", createdAt: "2026-10-09T00:00:00.000Z", cancelling: false, live: true, worker: true })); - return { deployments: [...listeners, ...running] } as T; - } - if (method === "GET" && url.pathname === "/api/tenants/t/assets") { - return assets.filter((asset) => asset.kind === url.searchParams.get("kind")) as T; - } - if (method === "POST" && path === "/api/tenants/t/assets") { - const asset = { id: `ast_${assets.length + 1}`, ...(body as { kind: string; name: string }) }; - assets.push(asset); - return asset as T; - } - if (method === "GET" && path.endsWith("/tarballs")) { - return [...tarballs].map(([filename, integrity]) => ({ filename, integrity, size: 1 })) as T; - } - if (method === "POST" && path === "/api/integrations/workflow-deploy/t") { - if (unconnected.includes((body as { entry: string }).entry)) throw new ApiError(409, "source_not_connected", "Connect it first."); - deployed.push(body); - const { definitionAssetId, pin } = body as { definitionAssetId: string; pin: string }; - const deployment: WorkflowDeployment = { - id: `run_${deployments.length + 1}`, - tenantId: "t", - definitionAssetId, - status: "deployed", - createdAt: `2026-10-09T00:00:0${deployments.length}.000Z`, - }; - deployments.push(deployment); - pins.set(deployment.id, pin); - return deployment as T; - } - if (method === "DELETE" && url.pathname.includes("/tarballs/")) { - tarballs.delete(decodeURIComponent(path.split("/").at(-1) ?? "")); - return { commit: "c" } as T; - } - if (method === "DELETE") { - const deployment = deployments.find((row) => path.endsWith(`/${row.id}`)); - if (deployment) deployment.status = "released"; - return undefined as T; - } - throw new Error(`unexpected ${method} ${path}`); - } - - async function raw(input: string | URL | Request, init?: RequestInit): Promise { - const path = String(input).replace(/^https?:\/\/[^/]+/, ""); - const method = init?.method ?? "GET"; - requests.push(`${method} ${path}`); - if (path === "/packages/index.json") return Response.json(INDEX); - if (method === "GET" && path.startsWith("/packages/")) return new Response(new Uint8Array([1])); - if (method === "PUT") { - const filename = decodeURIComponent(path.split("/").at(-1) ?? ""); - const integrity = INDEX.find((entry) => entry.filename === filename)?.integrity ?? ""; - tarballs.set(filename, integrity); - return Response.json({ commit: "c", integrity }); - } - throw new Error(`unexpected ${method} ${path}`); - } - - const transport: Transport = { fetch: json, subscribe: () => () => {} }; - return { transport, fetchRaw: raw as typeof fetch, assets, tarballs, deployments, pins, requests, deployed, workers }; -} - -function puts(requests: string[]): string[] { - return requests.filter((line) => line.startsWith("PUT ")).map((line) => decodeURIComponent(line.split("/").at(-1) ?? "")); -} - -describe("ensureWorkflows", () => { - test("publishes the tool before the workflows to one corbits registry and deploys each workflow pinned to its version", async () => { - const hub = fakeHub(); - expect(await ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw)).toEqual(["pr-triage-github", "pr-triage-bitbucket-datacenter", "pr-triage-historical"]); - - const registries = hub.assets.filter((asset) => asset.kind === "package-registry"); - expect(registries).toEqual([{ id: expect.any(String), kind: "package-registry", name: "corbits" }]); - const registryId = registries[0]!.id; - expect(hub.requests).toContain(`PUT /api/tenants/t/assets/${registryId}/tarballs/${encodeURIComponent(INDEX[0]!.filename)}`); - expect(puts(hub.requests)).toEqual(INDEX.map((entry) => entry.filename)); - function workflowAsset(name: string) { - return hub.assets.find((asset) => asset.kind === "workflow" && asset.name === name)?.id; - } - expect(hub.deployed).toEqual([ - { - registryAssetId: registryId, - definitionAssetId: workflowAsset("pr-triage-github"), - pin: "@corbits/pr-triage-github-workflow@0.1.0-sha-aaaaaaaa", - entry: "./pr-triage-github.mjs", - sourceOfferingIds: ["mof_1"], - defaultSourceOfferingId: "mof_1", - }, - { - registryAssetId: registryId, - definitionAssetId: workflowAsset("pr-triage-bitbucket-datacenter"), - pin: "@corbits/pr-triage-bitbucket-datacenter-workflow@0.1.0-sha-cccccccc", - entry: "./pr-triage-bitbucket-datacenter.mjs", - sourceOfferingIds: ["mof_1"], - defaultSourceOfferingId: "mof_1", - }, - { - registryAssetId: registryId, - definitionAssetId: workflowAsset("pr-triage-historical"), - pin: "@corbits/pr-triage-historical-workflow@0.1.0-sha-bbbbbbbb", - entry: "./pr-triage-historical.mjs", - sourceOfferingIds: ["mof_1"], - defaultSourceOfferingId: "mof_1", - }, - ]); - }); - - test("an unchanged build publishes and deploys nothing; a deployment of another version is redeployed though its tarball is already published", async () => { - const hub = fakeHub(); - await ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw); - hub.requests.length = 0; - expect(await ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw)).toEqual([]); - expect(puts(hub.requests)).toEqual([]); - - // A converge published this build, then its deploy failed: the live deployment still runs the older version. - hub.pins.set("run_1", "@corbits/pr-triage-github-workflow@0.1.0-sha-00000000"); - expect(await ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw)).toEqual(["pr-triage-github"]); - expect(puts(hub.requests)).toEqual([]); - expect(hub.deployments.map((row) => [row.id, row.status])).toEqual([ - ["run_1", "released"], - ["run_2", "deployed"], - ["run_3", "deployed"], - ["run_4", "deployed"], - ]); - }); - - test("deploys only the connected hosts' workflows, and skips one the hub reports is no longer connected", async () => { - const github = fakeHub(); - expect(await ensureWorkflows(github.transport, "t", GITHUB, MISSING, github.fetchRaw)).toEqual(["pr-triage-github", "pr-triage-historical"]); - - const disconnected = fakeHub({}, ["./pr-triage-bitbucket-datacenter.mjs"]); - expect(await ensureWorkflows(disconnected.transport, "t", BOTH, MISSING, disconnected.fetchRaw)).toEqual(["pr-triage-github", "pr-triage-historical"]); - }); - - test("refuses to republish a version whose bytes changed, before publishing or deploying anything", async () => { - const hub = fakeHub({ [INDEX[3]!.filename]: "sha512-other" }); - await expect(ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw)) - .rejects.toThrow("@corbits/pr-triage-historical-workflow@0.1.0-sha-bbbbbbbb is already published with different contents; this build did not pack reproducibly."); - expect(puts(hub.requests)).toEqual([]); - expect(hub.deployed).toEqual([]); - }); - - test("before publishing, removes every tarball neither this build nor a live listener or worker pins", async () => { - const hub = fakeHub(); - await ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw); - hub.pins.set("run_1", "@corbits/pr-triage-github-workflow@0.1.0-sha-00000000"); - hub.workers.push({ package: { name: "@corbits/pr-triage-github-workflow", version: "0.1.0-sha-22222222" }, tools: [{ name: "@corbits/github-tool", version: "0.1.0" }] }); - hub.tarballs.set("@corbits-pr-triage-github-workflow-0.1.0-sha-00000000.tgz", "sha512-listener"); - hub.tarballs.set("@corbits-pr-triage-github-workflow-0.1.0-sha-22222222.tgz", "sha512-worker"); - hub.tarballs.set("@corbits-github-tool-0.1.0.tgz", "sha512-worker-tool"); - hub.tarballs.set("@corbits-pr-triage-github-workflow-0.1.0-sha-11111111.tgz", "sha512-old"); - hub.tarballs.set("@corbits-github-tool-0.0.9.tgz", "sha512-old-tool"); - hub.tarballs.delete(INDEX[1]!.filename); - hub.requests.length = 0; - - expect(await ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw)).toEqual(["pr-triage-github"]); - const changes = hub.requests.filter((line) => /^(DELETE|PUT) .*\/tarballs\//.test(line) || line.includes("workflow-deploy")); - expect(changes.map((line) => decodeURIComponent(line))).toEqual([ - expect.stringMatching(/^DELETE .*\/@corbits-pr-triage-github-workflow-0\.1\.0-sha-11111111\.tgz$/), - expect.stringMatching(/^DELETE .*\/@corbits-github-tool-0\.0\.9\.tgz$/), - expect.stringMatching(/^PUT .*\/@corbits-pr-triage-github-workflow-0\.1\.0-sha-aaaaaaaa\.tgz$/), - "POST /api/integrations/workflow-deploy/t", - ]); - expect([...hub.tarballs.keys()].sort()).toEqual([ - ...INDEX.map((entry) => entry.filename), - "@corbits-pr-triage-github-workflow-0.1.0-sha-00000000.tgz", - "@corbits-pr-triage-github-workflow-0.1.0-sha-22222222.tgz", - "@corbits-github-tool-0.1.0.tgz", - ].sort()); - }); - - test("a host's code review deploys only while it is on, with an offering for each model it names", async () => { - const models = [ - ...DECISION, - { canonicalName: "claude-opus-5-5", offerings: [{ offeringId: "mof_opus", providerName: "anthropic" }] }, - { canonicalName: "claude-haiku-5-5", offerings: [{ offeringId: "mof_haiku", providerName: "anthropic" }] }, - ]; - const hub = fakeHub({}, [], models); - expect(await ensureWorkflows(hub.transport, "t", { connected: ["github"], codeReview: ["github"], codeReviewProvider: "anthropic", prWorkflow: [] }, MISSING, hub.fetchRaw)).toEqual(["pr-triage-github", "pr-triage-historical", "pr-code-review-github"]); - expect(hub.deployed.at(-1)).toMatchObject({ entry: "./pr-code-review-github.mjs", sourceOfferingIds: ["mof_opus", "mof_haiku"], defaultSourceOfferingId: "mof_opus" }); - - expect(await ensureWorkflows(hub.transport, "t", { connected: ["github"], codeReview: ["github"], codeReviewProvider: "anthropic", prWorkflow: [] }, { kinds: new Set(["code-review"]) }, hub.fetchRaw)).toEqual(["pr-code-review-github"]); - }); - - test("a host's PR workflow deploys while its provider's key is stored and code review has a place, whatever the repositories' switches", async () => { - const models = [ - ...DECISION, - { canonicalName: "claude-opus-5-5", offerings: [{ offeringId: "mof_opus", providerName: "anthropic" }] }, - { canonicalName: "claude-haiku-5-5", offerings: [{ offeringId: "mof_haiku", providerName: "anthropic" }] }, - ]; - const hub = fakeHub({}, [], models); - expect(await ensureWorkflows(hub.transport, "t", { ...GITHUB, prWorkflow: ["github"] }, MISSING, hub.fetchRaw)).toEqual(["pr-triage-github", "pr-triage-historical", "pr-workflow-anthropic-github"]); - expect(hub.deployed.at(-1)).toMatchObject({ entry: "./pr-workflow-github.mjs", sourceOfferingIds: ["mof_1", "mof_opus", "mof_haiku"], defaultSourceOfferingId: "mof_1" }); - - // Unknown leaves it; once no repository's policy file needs it, it is released. - await ensureWorkflows(hub.transport, "t", { ...GITHUB, prWorkflow: undefined }, MISSING, hub.fetchRaw); - expect(hub.requests.filter((line) => line.startsWith("DELETE") && line.includes("/workflows/runs/"))).toEqual([]); - await ensureWorkflows(hub.transport, "t", GITHUB, MISSING, hub.fetchRaw); - expect(hub.requests.filter((line) => line.startsWith("DELETE") && line.includes("/workflows/runs/"))).toEqual(["DELETE /api/tenants/t/workflows/runs/run_3"]); - }); - - test("a redeploy limited to one host deploys again only that host's workflows of the kinds named", async () => { - const hub = fakeHub(); - await ensureWorkflows(hub.transport, "t", BOTH, MISSING, hub.fetchRaw); - expect(await ensureWorkflows(hub.transport, "t", BOTH, { kinds: new Set(["triage", "pr-workflow"]), sources: ["bitbucket-datacenter"] }, hub.fetchRaw)).toEqual(["pr-triage-bitbucket-datacenter"]); - }); - - test("refuses to deploy a workflow whose model the workspace does not offer", async () => { - const hub = fakeHub({}, [], []); - await expect(ensureWorkflows(hub.transport, "t", GITHUB, MISSING, hub.fetchRaw)).rejects.toThrow("The workspace has no decision model. Save its key in Settings."); - expect(hub.deployed).toEqual([]); - }); -}); - -describe("cancelsSuperseded", () => { - test("pr-triage's superseded deployments are cancelled only right after a deploy; pr-triage-historical's on every converge", () => { - expect([cancelsSuperseded("pr-triage-github", true), cancelsSuperseded("pr-triage-github", false)]).toEqual([true, false]); - expect([cancelsSuperseded("pr-triage-historical", true), cancelsSuperseded("pr-triage-historical", false)]).toEqual([true, true]); - }); -}); diff --git a/apps/web/src/lib/workflow-deploy.ts b/apps/web/src/lib/workflow-deploy.ts index ce262ca1d..9a1399804 100644 --- a/apps/web/src/lib/workflow-deploy.ts +++ b/apps/web/src/lib/workflow-deploy.ts @@ -1,16 +1,6 @@ -// The hub bridge finds deployments by workflow name, so each workflow is -// redeployed only when its current deployment runs another version than this -// build's, none exists, or its model's key changed (a deployment resolves -// its offerings when deployed). -import { ApiError, listWorkflowDeployments, type Transport, type WorkflowDeployment } from "@intx/hub-client"; -import { requestOrigin } from "./hub-origin.ts"; -import { hubErrorMessage } from "./hub-transport.ts"; -import { currentDeployment, isLiveDeployment, listDeployedWorkflows, type DeployedWorkflow } from "./hub-api.ts"; -import { offeringsFor } from "./model-catalog.ts"; -import { tarballFilename, type CodeReviewProvider, type RepoSource } from "@corbits/triage-contracts"; -import { WORKFLOW_PACKAGES, type PackageIndexEntry, type WorkflowKind, type WorkflowPackage } from "./workflow-packages.ts"; - -const REGISTRY_NAME = "corbits"; +// Shared by publishing and upgrading: workflow assets, the portal build's files, a workspace's deployment lifetime and +// cancelling a deployment. +import { ApiError, type Transport } from "@intx/hub-client"; type Asset = { id: string; kind: string; name: string; creatorPrincipalId: string | null }; @@ -27,80 +17,16 @@ export async function portalPackage(fetchRaw: typeof fetch, file: string): Promi return response; } -async function putTarball(fetchRaw: typeof fetch, tenantId: string, registryId: string, entry: PackageIndexEntry, bytes: ArrayBuffer): Promise { - const response = await fetchRaw(`${requestOrigin()}${tarballPath(tenantId, registryId, entry.filename)}`, { - method: "PUT", - credentials: "include", - headers: { "content-type": "application/octet-stream" }, - body: bytes, - }); - if (response.ok) return; - let detail: { error?: { code?: string; message?: string } } = {}; - try { detail = await response.json() as typeof detail; } catch { /* generic error below */ } - throw new ApiError(response.status, detail.error?.code ?? "unknown", hubErrorMessage(response.status, detail)); -} - -function tarballPath(tenantId: string, registryId: string, filename?: string): string { - const base = `/api/tenants/${encodeURIComponent(tenantId)}/assets/${encodeURIComponent(registryId)}/tarballs`; - return filename === undefined ? base : `${base}/${encodeURIComponent(filename)}`; -} - -/** - * Interchange delivers a workflow the registry's whole tip tree under a size - * cap, so the registry keeps only this build's tarballs and those a live - * deployment's closure pins; the asset's git history keeps the rest. Runs - * before anything is published or deployed, so a registry already over the - * cap shrinks before this build's deploy is probed. A version is never - * overwritten, and a tarball already removed by another converge is skipped. - */ -async function syncRegistry( - transport: Transport, - fetchRaw: typeof fetch, - tenantId: string, - registryId: string, - index: PackageIndexEntry[], - deployments: readonly DeployedWorkflow[], -): Promise { - const listed = await transport.fetch>("GET", tarballPath(tenantId, registryId)); - const missing: PackageIndexEntry[] = []; - for (const entry of index) { - const existing = listed.find((tarball) => tarball.filename === entry.filename); - if (!existing) missing.push(entry); - else if (existing.integrity !== entry.integrity) { - throw new Error(`${entry.name}@${entry.version} is already published with different contents; this build did not pack reproducibly.`); - } - } - const kept = new Set(index.map((entry) => entry.filename)); - for (const deployment of deployments) { - for (const pkg of [deployment.package, ...deployment.tools]) if (pkg) kept.add(tarballFilename(pkg)); - } - for (const { filename } of listed) { - if (!kept.has(filename)) await deleteTarball(transport, tenantId, registryId, filename); - } - // In index order, so the tools and native modules land before the workflows that depend on them. - for (const entry of missing) { - await putTarball(fetchRaw, tenantId, registryId, entry, await (await portalPackage(fetchRaw, entry.filename)).arrayBuffer()); - } -} - -async function deleteTarball(transport: Transport, tenantId: string, registryId: string, filename: string): Promise { - try { - await transport.fetch("DELETE", tarballPath(tenantId, registryId, filename)); - } catch (error) { - if (!(error instanceof ApiError && error.status === 404)) throw error; - } -} - /** The longest lifetime Interchange accepts, so a deployment lives until a newer one replaces it. */ const KEEP_DEPLOYED_LIFETIME = "36500d"; /** * Interchange creates a workflow's definition during its first deploy, so a * per-definition lifecycle cannot apply to that deployment. The tenant only - * runs triage workflows, so its ceiling keeps every triage listener - * (pr-triage and pr-triage-historical) deployed until a newer one replaces it. + * runs triage workflows, so its ceiling keeps every listener deployed until a + * newer one replaces it. */ -async function ensureKeepDeployedLifetime(transport: Transport, tenantId: string): Promise { +export async function ensureKeepDeployedLifetime(transport: Transport, tenantId: string): Promise { const path = `/api/tenants/${encodeURIComponent(tenantId)}`; const { config } = await transport.fetch<{ config?: { lifecycle?: { maxLifetime?: string } } }>("GET", path); if (config?.lifecycle?.maxLifetime === KEEP_DEPLOYED_LIFETIME) return; @@ -115,112 +41,3 @@ export async function cancelDeployment(transport: Transport, tenantId: string, d if (!(error instanceof ApiError && error.status === 409)) throw error; } } - -function newestFirst(a: WorkflowDeployment, b: WorkflowDeployment): number { - return b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id); -} - -async function cancelSuperseded(transport: Transport, tenantId: string, assetId: string): Promise { - const deployments = await listWorkflowDeployments(transport, tenantId); - const ofAsset = deployments.filter((d) => d.definitionAssetId === assetId).sort(newestFirst); - for (const old of ofAsset.slice(1)) { - if (isLiveDeployment(old.status)) await cancelDeployment(transport, tenantId, old.id); - } -} - -/** - * The hub replaces a busy pr-triage deployment with a copy and releases the - * old one itself, so pr-triage's superseded deployments are cancelled here - * only right after a deploy; other workflows' are cancelled on every converge, - * which retries a failed cancel. - */ -export function cancelsSuperseded(workflow: string, deployed: boolean): boolean { - return deployed || workflow === "pr-triage-historical"; -} - -/** The hub refuses a host's workflow while that host is not connected, which can change after the portal read it. */ -function sourceNotConnected(cause: unknown): boolean { - return cause instanceof ApiError && cause.status === 409 && cause.code === "source_not_connected"; -} - -/** - * The hosts the workspace connected, those whose code review is on, the provider it runs code review on, and the hosts - * whose PR workflow deploys, undefined when the hub did not say. - */ -export type DeployHosts = { connected: readonly RepoSource[]; codeReview: readonly RepoSource[]; codeReviewProvider: CodeReviewProvider | undefined; prWorkflow: readonly RepoSource[] | undefined }; - -function wanted(workflow: WorkflowPackage, hosts: DeployHosts): boolean { - if (!hosts.connected.includes(workflow.source)) return false; - if (workflow.kind === "triage") return true; - if (workflow.provider !== hosts.codeReviewProvider) return false; - if (workflow.kind === "pr-workflow") return hosts.prWorkflow?.includes(workflow.source) === true; - return hosts.codeReview.includes(workflow.source); -} - -/** The PR workflow follows the policy files: once none needs it on a host, or its key is gone, its deployment goes. */ -async function releaseUnwanted(transport: Transport, tenantId: string, hosts: DeployHosts, deployments: readonly DeployedWorkflow[]): Promise { - if (hosts.prWorkflow === undefined) return; - const unwanted = new Set(WORKFLOW_PACKAGES.filter((workflow) => workflow.kind === "pr-workflow" && !wanted(workflow, hosts)).map((workflow) => workflow.name)); - for (const deployment of deployments) { - if (unwanted.has(deployment.workflow) && !deployment.cancelling && !deployment.worker) await cancelDeployment(transport, tenantId, deployment.id); - } -} - -/** The kinds deployed again even at the same version, on `sources` only when it is given. */ -export type Redeploy = { kinds: ReadonlySet; sources?: readonly RepoSource[] }; - -function redeploys(redeploy: Redeploy, workflow: WorkflowPackage): boolean { - return redeploy.kinds.has(workflow.kind) && (redeploy.sources === undefined || redeploy.sources.includes(workflow.source)); -} - -/** Each host's triage deploys while the host is connected, its code review while that is on too, its PR workflow while a repository's policy file needs it; `redeploy` names those deployed again even at the same version. */ -export async function ensureWorkflows( - transport: Transport, - tenantId: string, - hosts: DeployHosts, - redeploy: Redeploy, - fetchRaw: typeof fetch = fetch, -): Promise { - const deployments = await listDeployedWorkflows(transport, tenantId); - await ensureKeepDeployedLifetime(transport, tenantId); - const registry = await ensureAsset(transport, tenantId, "package-registry", REGISTRY_NAME); - const index = await (await portalPackage(fetchRaw, "index.json")).json() as PackageIndexEntry[]; - await syncRegistry(transport, fetchRaw, tenantId, registry.id, index, deployments); - const deployed: string[] = []; - for (const workflow of WORKFLOW_PACKAGES.filter((row) => wanted(row, hosts))) { - const pkg = index.find((entry) => entry.name === workflow.packageName); - if (!pkg) throw new Error(`The portal build is missing the ${workflow.name} workflow.`); - // The deploy route needs a workflow asset to name the definition; the code comes from the registry. - const asset = await ensureAsset(transport, tenantId, "workflow", workflow.name); - const current = currentDeployment(deployments, workflow.name)?.package; - const deploy = redeploys(redeploy, workflow) || current?.name !== pkg.name || current.version !== pkg.version; - if (deploy) { - const offerings = await offeringsFor(transport, tenantId, workflow.models); - try { - await transport.fetch("POST", `/api/integrations/workflow-deploy/${encodeURIComponent(tenantId)}`, { - registryAssetId: registry.id, - definitionAssetId: asset.id, - pin: `${pkg.name}@${pkg.version}`, - entry: workflow.entry, - sourceOfferingIds: offerings.ids, - defaultSourceOfferingId: offerings.defaultId, - }); - } catch (cause) { - if (sourceNotConnected(cause)) continue; - throw cause; - } - deployed.push(workflow.name); - } - if (cancelsSuperseded(workflow.name, deploy)) await cancelSuperseded(transport, tenantId, asset.id); - } - await releaseUnwanted(transport, tenantId, hosts, deployments); - return deployed; -} - -/** Releases the live deployments of one kind of workflow, such as code review once the Anthropic key it runs on is removed. */ -export async function releaseWorkflows(transport: Transport, tenantId: string, kind: WorkflowKind): Promise { - const names = new Set(WORKFLOW_PACKAGES.filter((workflow) => workflow.kind === kind).map((workflow) => workflow.name)); - for (const deployment of await listDeployedWorkflows(transport, tenantId)) { - if (names.has(deployment.workflow) && !deployment.cancelling) await cancelDeployment(transport, tenantId, deployment.id); - } -} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ec59b26d4..c0082e577 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -90,25 +90,6 @@ GitHub ──webhook──► Hub ──spawn──► Sidecar ──► pr-tria within the last two minutes; one deployed without a sidecar past that is released, so the next Upgrade deploys it again. The workflow-versions route reports each deployment's `live` the same way, which Settings reads. - - **Coordinator**: every standalone code review mail is a job in one in-memory queue keyed per lane (triage or - code review), host and pull request. Each job runs in a worker deployment - of its own, copied from the host's live deployment, which only seeds - workers, and released once its run ends. How many run at once is the - provisioner's call: a placement it declines for want of room waits and is - retried with backoff. A job with no live deployment to copy, or whose - worker the sender gate refuses (a copy of a deployment made before its - workflow declared who may mail it, which is released), is retried the same - way five times between them, enough for the portal reload after a hub - deploy and usually enough for a failed deployment's heal; then the head - fails with "this repository's Triage deployment is not running; open the - portal to redeploy it" or "this repository's Triage deployment is out of - date; open the portal to redeploy it", and the reconciler queues it again - once a deployment is live. A worker not finished an hour after its spawn - began, stuck readying, delivering or running, is cancelled and its slot - freed; the reconciler retries its head as a cancelled run. The queue, like - the coalescer's windows, lives in memory, so the hub runs as one replica. - A restart loses only queue order: the hub releases orphaned workers on - start and the reconciler queues their heads again. - **Run recorder**: reads the live logs of tenants with a run in flight and records each verdict and executed action. - **App Manifest conversion**: GitHub returns a one-time code that must be @@ -185,9 +166,9 @@ GitHub ──webhook──► Hub ──spawn──► Sidecar ──► pr-tria (`no_isolated_place`), nothing is mailed, the host's code review deployments are released, and the portal links Settings → Code review in place of the switch. -- **Portal** (`apps/web`): owns setup. After sign-in it converges the tenant, - credentials, inference offerings, workflow sources and deployments over - stock hub routes. The hub seeds nothing for it. It is a wrapper over the +- **Portal** (`apps/web`): owns setup. It writes the tenant, credentials and + inference offerings over stock hub routes, and **Upgrade to new version** + publishes the workflow sources and deploys the listeners. The hub seeds nothing for it. It is a wrapper over the routes; nothing is possible only through it. Each minute and on focus it reads the served `index.html`; once its hashed entry script differs from the one the tab loaded, it offers a reload, since a stale tab would deploy the @@ -204,10 +185,8 @@ any other tenant or an unknown sender. - `pr-triage-github` and `pr-triage-bitbucket-datacenter`: one mail names one pull request and triages it. Both are built by `prTriage({ sources })` in - `@corbits/triage-workflows`. Each binds only its host's credential, since an - unresolved binding fails a deployment, so a workspace deploys one per - connected host. A deployment of the earlier `pr-triage` workflow is released - once `pr-triage-github` is live. + `@corbits/triage-workflows`. Nothing deploys them; Upgrade cancels any + still live. - `pr-triage-historical`: one run that triages every open pull request of a GitHub repository, skipping drafts. The hub no longer mails it: its backlog route, run when triage is turned on, when a push to the default branch @@ -221,8 +200,7 @@ any other tenant or an unknown sender. names one pull request and the run returns a review. - `pr-workflow-anthropic-github` and `pr-workflow-anthropic-bitbucket-datacenter`: triage, then code review behind a gate, in one run ([PR workflow](#pr-workflow)). - The portal deploys a host's while a repository's policy file needs it; the - hub picks it per pull request. + Nothing deploys them; Upgrade cancels any still live. - `triage-listener-github` and `triage-listener-bitbucket-datacenter` (`triageListener` in `@corbits/triage-manager`): a workspace's listener for one host. The hub mails it the pull requests it recorded that have no @@ -242,14 +220,25 @@ any other tenant or an unknown sender. deploy route from that person's source asset, waits for it to be live (cancelling it after five minutes), then cancels the host's live listeners created before it, found by workflow name, so of two upgrades at once the - newer stays. Settings offers Upgrade again while a host has more than one + newer stays, and every live deployment of the workflows the listener + replaced: pr-triage, pr-triage-historical, the standalone code review and + the PR workflow variants. Settings offers Upgrade again while a host has more than one live listener. A published workflow's asset, and so its definition, is named for its publisher and release, so the hub finds a deployment by the workflow id in its frozen projection (`apps/hub/src/github/deployment.ts`), and the - workflow-versions route reports a source deployment's package from its + workflow-versions route reports a source deployment's asset, commit and package from its closure. Onboarding runs the same upgrade once the decision model is saved, - for an owner or Deployer. + for an owner or Deployer. The upgrade also sets the tenant's + `lifecycle.maxLifetime` to the longest Interchange accepts, so a listener + lives until a newer one replaces it. A host's listener and its managers are + released once the host is disconnected: right away when Bitbucket Data + Center is disconnected through its route, and on the next reconcile pass + when the GitHub App credential is removed. A running listener keeps the + credentials it launched with, so after a host's credential is replaced the + portal deploys that host's newest listener again from the same asset + commit, which the workflow-versions route reports, and cancels the older + ones, as Upgrade does; managers and judges launch with the current value. - `triage-manager-github` and `triage-manager-bitbucket-datacenter` (`triageManager`): one live run per pull request, which runs only while it has work, and the only workflow that writes to the pull request. It takes @@ -500,14 +489,11 @@ operation has one implementation. repository; it needs the host connected and a code review provider's key. A workspace runs one provider, the one whose key it stores in Settings → Code review -(`CODE_REVIEW_PROVIDERS` in `triage-contracts`; only Anthropic today). The -portal deploys that host's `pr-code-review-` from the provider's -package, and the hub releases it once no repository has code review on, the -host is disconnected or the key is gone. - -Each new non-draft head of a pull request in such a repository is mailed once, -whether or not triage is on, with the host's unified diff cut at a line -boundary past 100,000 characters; on GitHub a diff too large for the diff API +(`CODE_REVIEW_PROVIDERS` in `triage-contracts`; only Anthropic today). A +pull request's code review runs in its manager: the hub attaches the review +request to a pull request mail whose `codeReview` is on, which the policy +file's `code-review` block or Review again sets, with the host's unified diff +cut at a line boundary past 100,000 characters; on GitHub a diff too large for the diff API is built from the files' patches. The files a cut leaves unread, read from the diff's headers, ride as `cut.unread` and never reach a prompt; the review sent to GitHub lists them. The title and description are read as the facts step @@ -543,10 +529,9 @@ Each request carries `diffHash`, a SHA-256 of the full diff, `index` lines included, so a base change to a file the diff touches, or a binary change, changes it. A diff with no `index` line, as GitHub's rebuilt from the file list, is sent unhashed. A manager's code review reports the hash with its -review, so carrying over works after a restart: a new head whose diff hashes the same as the newest review's, -when that review finished, is not mailed but takes the review over (`carries` -in `code-review-runs.ts`). `GET …/code-review` with `head` answers `carriedTo`, -reading that head's diff once; a failed read is not retried for five minutes. +review. `GET …/code-review` with `head` answers `carriedTo` when that head's +diff hashes the same as the newest finished review's, reading that head's diff +once; a failed read is not retried for five minutes. The review is then sent to GitHub at that head, unless it was sent on its own, whose comments stay on the same lines. A diff that can't be read and `…/run` review the head instead. @@ -701,9 +686,8 @@ token, never the App key. The vault holds the App as webhook secret under `github-hook`. The manifest callback and `PUT /api/integrations/github-app/:tenantId` write the key and the token minted from it in one transaction; App JSON a stock credential write leaves in -`github` is moved to `github-app` when the hub next reads the key: the next worker deploy, portal read, installation sync or token loop pass, so at most 45 minutes later; until then that credential holds App JSON and a deployment launched from it is handed the key. The hub -mints the token before it deploys a GitHub worker (when under 15 minutes are -left) and renews it on boot and every 45 minutes. Every write holds the +`github` is moved to `github-app` when the hub next reads the key: the next portal read, installation sync or token loop pass, so at most 45 minutes later; until then that credential holds App JSON and a deployment launched from it is handed the key. The hub +mints the token on boot and renews it every 45 minutes. Every write holds the tenant's advisory lock, and a renewal stores its token only while the key it minted from is still the workspace's, so a replaced App is never overwritten with the old one's token. Each renewal then calls stock @@ -716,7 +700,7 @@ key whose `github` credential was deleted is dropped by the renewal loop. A failed renewal keeps its reason as `tokenError` on the credential's metadata, shown on the portal's GitHub App row; once the token is past its expiry the credential is marked `expired`, which stock launch resolution skips, so no -worker deploys with it. The next good renewal sets it `active` again; running +deployment launches with it. The next good renewal sets it `active` again; running deployments still re-resolve an expired credential, as stock drops only revoked ones. Hub routes that reach GitHub for the portal sign with the App key on the hub. @@ -883,11 +867,11 @@ read it, and passes the rest of the request to stock. the coalescer's mail carries, so a redelivery records nothing new; any other mail is a new event, so a head mailed again, as after its base moved or its CI settled, - is triaged again. The code review feed sends nothing. A workspace with no - live listener for the host mails nothing: Triage again, Review again, a - held Do's approval and backlog triage answer 503 `service_not_running`, and - a webhook's head fails, each saying Triage is stopped on that host until - someone presses **Upgrade to new version** in Settings. + is triaged again. A workspace with no live listener for the host mails + nothing: Triage again, Review again, a held Do's approval and backlog + triage answer 503 `service_not_running`, and a webhook's head fails, each + saying Triage is stopped on that host until someone presses **Upgrade to new + version** in Settings. - Webhook replay protection uses the `@corbits/webhooks` table and a delivery-id cache in `HUB_DATA_DIR`. diff --git a/docs/BITBUCKET_DATACENTER.md b/docs/BITBUCKET_DATACENTER.md index e64b5ca0b..64e8f8202 100644 --- a/docs/BITBUCKET_DATACENTER.md +++ b/docs/BITBUCKET_DATACENTER.md @@ -29,7 +29,7 @@ workflow, the same steps as `pr-triage-github` built over the Bitbucket tools `bitbucket-datacenter` credential, so the hub deploys it once the instance is connected (`POST /api/integrations/workflow-deploy` answers 409 `source_not_connected` before that); a workspace with GitHub as well runs both. -Disconnecting releases the deployment and leaves the Bitbucket repository records disabled and disconnected, so the open pull request listing carries on with the other hosts. The webhook credential names this +Disconnecting releases the host's listener and its managers and leaves the Bitbucket repository records disabled and disconnected, so the open pull request listing carries on with the other hosts. The webhook credential names this workflow, and every mail carries `repoSource: "bitbucket-datacenter"`. Triage with every rule switched off starts no run and writes nothing to the diff --git a/docs/Corbits Triage User Manual/Corbits Triage User Manual.md b/docs/Corbits Triage User Manual/Corbits Triage User Manual.md index 1a00300b9..b5c6eca2e 100644 --- a/docs/Corbits Triage User Manual/Corbits Triage User Manual.md +++ b/docs/Corbits Triage User Manual/Corbits Triage User Manual.md @@ -26,7 +26,7 @@ Think of it as a first reader. It does the routine checking (Did the tests pass? ### What it never does on its own -Nothing runs until you turn it on. Connecting GitHub or Bitbucket Data Center only records your repositories, with triage switched off. A repository is checked only after someone turns on **Triage pull requests in this repository**. Enabling a repository switches on Comment and Labels for it; every other action waits for a person to click until you decide which actions Triage may run by itself, and in which repositories. Code review, too, runs only in repositories where someone switched it on or whose [policy file](#writing-the-file) turns it on, and it writes to the pull request only when someone sends it. +Nothing runs until you turn it on. Connecting GitHub or Bitbucket Data Center only records your repositories, with triage switched off. A repository is checked only after someone turns on **Triage pull requests in this repository**. Enabling a repository switches on Comment and Labels for it; every other action waits for a person to click until you decide which actions Triage may run by itself, and in which repositories. Code review, too, runs only in repositories whose [policy file](#writing-the-file) turns it on, and it writes to the pull request only when someone sends it. ### The three parts @@ -38,7 +38,7 @@ Nothing runs until you turn it on. Connecting GitHub or Bitbucket Data Center on Code review reads each new commit of a pull request for bugs: mistakes in the changed lines and breakage the change causes elsewhere in the repository. It is separate from the verdict and optional. -- **Turning it on.** Switch on **Review the code in every pull request for bugs** in each repository's [Code review section](#code-review-1). When no key is saved, it asks for the workspace's Anthropic API key; you can also save it under [Settings → Code review](#code-review-2). Switch it off there to stop. A [policy file](#writing-the-file) can turn it on instead: it then runs after triage, for the groups in `for` and on the verdicts and paths in `when`, and the switch no longer decides. +- **Turning it on.** A repository's [policy file](#writing-the-file) turns it on: it then runs after triage, for the groups in `for` and on the verdicts and paths in `when`. It needs the workspace's Anthropic API key, saved under [Settings → Code review](#code-review-2). - **What it costs.** Each commit it reviews is billed to your Anthropic account. It reviews a pull request when it is opened and after every push while it is not a draft, and **Review again** is one more review. A push that leaves the change exactly as it was reviewed, such as a rebase onto a newer base that changed none of the files it touches, is not reviewed again: the new commit keeps the last review, and the section says **Carried over from** the commit it was made for. **Review again** reviews it anyway. - **Where you see it.** In the Inbox's reading pane, under **Code review**: each bug with how severe it is, its file and line, and the code it points at. **Review again** reviews the latest commit once more. A very large change is read up to a size limit, and the review says how much of it was read and lists the files it did not review in full (a long list starts collapsed), or says some files were not reviewed when it cannot tell which. A review that stopped before it finished reading the repository says **Stopped early: the reviewer reached its limit for reading the repository.** - **Sending it to GitHub.** On a GitHub pull request, **Send to GitHub** posts the review as one review from the Triage App: each finding on a changed line is a comment on that line, and the rest are listed in the review. A finding whose lines run past the changed ones sits on its first changed lines and names the others. A review that stopped early carries the same line in the posted review, and a review of a cut change lists the files it did not review in full. A review too long for GitHub lists what fits and says how many more are in the portal. It is sent once per commit; the section then says **Sent to GitHub**, and sending it again only tells you it is already there. A review carried over to a newer commit is sent on that commit, unless it was already sent on the earlier one, where its comments stay on the same lines. A review of a newer commit is sent as a new review, and the earlier one stays. @@ -202,7 +202,7 @@ decide: - { if: { code-review: findings }, do: [{ do: request-review, target: { to: codeowners } }] } ``` -Triage checks these keys when it reads the file. A file that turns a check on, or writes its own `decide`, runs triage, then its checks, then `decide` once to choose what happens, and the file alone then decides code review for that repository, whatever its Code review switch says. Only `code-review` with provider `anthropic` runs as a check today: it reviews each pull request's code after triage, for the groups in `for` and on the verdicts and paths in `when`. By default it reviews only once the rules in your file that fail a pull request and need no model have passed. CI counts only when `ci-green` is on. Triage's built-in checks, such as a draft or a requested change, don't hold the review back. A pull request that fails one gets its Fail with those reasons and no review, and one whose CI is still running is reviewed once CI passes. Write `when: { rules: any }` to review whatever those rules say. When triage could not read the pull request, code review waits instead of running. A check's `when: { paths: [...] }` limits it to pull requests where a changed file matches one of the globs. A glob starting with `!` leaves out the files it matches, so the `code-review` example above skips pull requests that change only Markdown docs or lockfiles (`*.lock`, which covers `bun.lock`, plus `package-lock.json` and `pnpm-lock.yaml`). A renamed file counts under its old and new path. When Triage can't read every changed file, the check is recorded as unfinished, never skipped. Such a file needs the workspace's Anthropic key ([Settings → Code review](#code-review-2)) and a [place to run](#where-code-review-runs), even one that only writes `decide`. Until both are set up, or while the file turns on a check that can't run yet, such as `security`, triage runs alone and approve and merge wait for you in the Inbox. Once no repository's file needs it, it stops running. On GitHub, a pull request carrying the `pauseLabel` label starts no triage or check run while it has that label, and removing the label triages it again. On GitHub, while a pull request in a repository with a `pauseLabel` can't be read, it isn't triaged and is tried again later, never run as if unpaused. Through a long GitHub outage it is retried a limited number of times, then triaged on its next event. `pace` doesn't act yet. +Triage checks these keys when it reads the file. A file that turns a check on, or writes its own `decide`, runs triage, then its checks, then `decide` once to choose what happens, and the file alone decides code review for that repository. Only `code-review` with provider `anthropic` runs as a check today: it reviews each pull request's code after triage, for the groups in `for` and on the verdicts and paths in `when`. By default it reviews only once the rules in your file that fail a pull request and need no model have passed. CI counts only when `ci-green` is on. Triage's built-in checks, such as a draft or a requested change, don't hold the review back. A pull request that fails one gets its Fail with those reasons and no review, and one whose CI is still running is reviewed once CI passes. Write `when: { rules: any }` to review whatever those rules say. When triage could not read the pull request, code review waits instead of running. A check's `when: { paths: [...] }` limits it to pull requests where a changed file matches one of the globs. A glob starting with `!` leaves out the files it matches, so the `code-review` example above skips pull requests that change only Markdown docs or lockfiles (`*.lock`, which covers `bun.lock`, plus `package-lock.json` and `pnpm-lock.yaml`). A renamed file counts under its old and new path. When Triage can't read every changed file, the check is recorded as unfinished, never skipped. Such a file needs the workspace's Anthropic key ([Settings → Code review](#code-review-2)) and a [place to run](#where-code-review-runs), even one that only writes `decide`. Until both are set up, or while the file turns on a check that can't run yet, such as `security`, triage runs alone and approve and merge wait for you in the Inbox. Once no repository's file needs it, it stops running. On GitHub, a pull request carrying the `pauseLabel` label starts no triage or check run while it has that label, and removing the label triages it again. On GitHub, while a pull request in a repository with a `pauseLabel` can't be read, it isn't triaged and is tried again later, never run as if unpaused. Through a long GitHub outage it is retried a limited number of times, then triaged on its next event. `pace` doesn't act yet. An action's `when: { author: [...] }`, a check's `for` and a `decide` rule's `author` can name a group. An author is in the first of your groups they match, in the order you wrote them, and in each built-in group (`maintainers`, `external`, `first-timers`, `bots`) their role puts them in. Everyone is in `everyone`. The built-in groups go by the author's role alone, so they still work when Triage can't read teams; only when it can't tell the author's role are they unknown too. For a group you define, Triage can't tell whether the author is in it when it can't tell their role, or when teams it can't read would decide which of your groups is theirs. An action for a group Triage can't tell about doesn't run. `everyone` always matches. Never let Approve or Merge rest on `everyone` alone. In `decide`, `{ all: clean }` means at least one check ran and every check that ran came back clean; checks that were skipped don't count, so when every check was skipped it doesn't match. A rule that approves or merges without naming `triage` only matches a Pass. A check with `clean: { below: medium }` is still clean when its only findings are less severe than medium. A review that couldn't read the whole change is incomplete, and incomplete is never clean. A file that uses `extends` can't hold these keys. Every key is listed in the setup skill's [reference](../../skills/corbits-triage-setup/reference.md#blocks-and-decide). @@ -352,17 +352,15 @@ On Bitbucket Data Center labels have no effect; see [Bitbucket Data Center repos ### Code review -Code review reads each new commit of a pull request for bugs. It runs whether or not triage is on for the repository, and skips drafts. Switch on **Review the code in every pull request for bugs** to start it, and off to stop. In a repository whose [policy file](#writing-the-file) turns a check on or writes its own `decide`, the file decides instead: changing the switch says **This repository's policy file decides code review, which runs with triage.** **Review again** still reviews the latest commit once more. +Code review reads each new commit of a pull request for bugs, and skips drafts. It runs with triage, so only in a repository where triage is on, and only when the repository's [policy file](#writing-the-file) turns code review on. **Review again** reviews the latest commit once more, whatever the policy file says. -![The popup Code review needs an Anthropic key, with the Anthropic key field, You won't see it again, Cancel and Save and turn on](images/27.png) - -It needs the workspace's Anthropic key. If none is saved yet, switching it on opens **Code review needs an Anthropic key**: paste the key and press **Save and turn on**, and code review is on. You won't see the key again. **Cancel** leaves it off, and the popup cannot be closed while the key is saving. The key is saved for the whole workspace, so other repositories use it too, and [Settings → Code review](#code-review-2) shows it. If you are not allowed to see the workspace's keys, the switch stays off and the section asks you to have an owner add the key; if Triage could not read the keys, the section says **Could not read the workspace's keys** with the reason, and the switch stays off until they can be read. +It needs the workspace's Anthropic key, saved under [Settings → Code review](#code-review-2). ### Where code review runs Code review checks out the pull request's code, so each review runs in a place of its own, apart from Triage and from other reviews: a Docker container on the server that runs Triage, or on a separate machine such as a VPS ([Docker on another machine](../provisioners/docker-remote.md) says how your server administrator sets that up), or in an [E2B](../provisioners/e2b.md) or [Railway](../provisioners/railway-sandboxes.md) sandbox in the cloud. [Settings → Code review](#code-review-2) says where it runs. -A server with no isolated place set up for Triage has nowhere to run it. The switch is then replaced by **Code review needs an isolated place to run**, with **Set one up**, which opens Settings → Code review, and **Review again** is hidden in the Inbox. A repository that had code review on says **It is switched on here but not running**, with **Turn off**; nothing is reviewed until a place is set up, and then it carries on without being switched on again. +A server with no isolated place set up for Triage has nowhere to run it. Each repository's page then says **Code review needs an isolated place to run**, with **Set one up**, which opens Settings → Code review, and **Review again** is hidden in the Inbox. Nothing is reviewed until a place is set up. ![The Code review section on a server with no isolated place: Code review needs an isolated place to run, with Set one up](images/38.png) @@ -494,7 +492,7 @@ In the new version, each pull request gets one manager that posts a single updat - **Triage is stopped until it is upgraded.** That host has not switched yet, so nothing is triaged there until someone presses **Upgrade to new version**. - **Triage runs version 0.1.0-sha-66ee60f0, published by Ana.** That host runs the new version, published by that person. When this portal has a newer release, it adds **A new version is ready.** If an earlier copy is somehow still running beside it, it adds **Another copy is still running too; upgrade to stop it.** -Owners and Deployers (see [People](#people)) see **Upgrade to new version** whenever an upgrade is due. Pressing it publishes the new version under your name, starts it on each host, and then stops the versions started before it, whoever published them. If two people upgrade at once, the newer start stays, and the other person sees **Someone else's upgrade finished first; Triage runs their version.** The button says what it is doing. If the new version does not start within five minutes, it is stopped, the one already running keeps running, and the error names the host. Everyone else sees **Ask a workspace owner or a Deployer to upgrade.** **Advanced info** lists this portal's release, who published the running one and when, and the version of each workflow in the release. +Owners and Deployers (see [People](#people)) see **Upgrade to new version** whenever an upgrade is due. Nothing starts Triage on a host by itself: a host connected after the last upgrade, or a decision model saved in Settings, waits for someone to press it. Pressing it publishes the new version under your name, starts it on each host, and then stops the versions started before it, whoever published them, along with the earlier triage and code review workflows it replaces. If two people upgrade at once, the newer start stays, and the other person sees **Someone else's upgrade finished first; Triage runs their version.** The button says what it is doing. If the new version does not start within five minutes, it is stopped, the one already running keeps running, and the error names the host. Everyone else sees **Ask a workspace owner or a Deployer to upgrade.** **Advanced info** lists this portal's release, who published the running one and when, and the version of each workflow in the release. During first-time setup, saving the decision model starts the new version the same way when you are an owner or a Deployer. @@ -532,11 +530,11 @@ Lists each connected repository and whether its default branch has a `.corbits-t **Judge pull requests with** names the decision model in use, by its provider, or by its address and model name for your own provider. It answers your policies' Judgement rules and chooses the verdict. -The row says **Key stored** or **No key**. With no key, its form is open below it: pick one of the listed providers and paste its API key, or choose **Add my own provider** and enter the address of a compatible service. **Change** opens the form again to replace the key, and **Remove** deletes it after you confirm. Keys are kept in the server's vault and never shown again. Every save of the decision model redeploys Triage's workers, so give it a minute before expecting new verdicts. +The row says **Key stored** or **No key**. With no key, its form is open below it: pick one of the listed providers and paste its API key, or choose **Add my own provider** and enter the address of a compatible service. **Change** opens the form again to replace the key, and **Remove** deletes it after you confirm. Keys are kept in the server's vault and never shown again. #### Rolling credentials -Every secret Triage holds can be replaced from Settings without disconnecting anything. After a GitHub or Bitbucket Data Center secret is replaced, Triage restarts its workers for that host, and code review uses the new value from its next review. If the restart fails, the row says **Saved. Restarting Triage failed; try again** with a **Try again** button; the new secret is already saved. +Every secret Triage holds can be replaced from Settings without disconnecting anything. After you replace a credential, Triage restarts its listener and uses the new value; pull requests in progress finish with the old one. If the restart fails, the row says **Saved. Restarting Triage failed; try again** with a **Try again** button; the new secret is already saved. **GitHub App key.** Press **Replace key** on the GitHub App row. In your App's settings on GitHub, generate a new private key, then choose the downloaded `.pem` file and press **Save**; GitHub checks that the key belongs to the connected App before it is saved. Then delete the old key on GitHub. To switch to another App, use **Create GitHub App**. @@ -546,7 +544,7 @@ Every secret Triage holds can be replaced from Settings without disconnecting an **Bitbucket Data Center webhook secret.** Press **Rotate secret** on the webhook secret row. Triage saves a new secret and updates every webhook it created to use it. Repositories whose webhook was set up by hand, or whose webhook Triage could not update, are named afterwards: each one's repository page shows the new secret, so update its webhook in Bitbucket from there. Pull requests that change in the meantime are picked up automatically. -**Model keys.** Press **Change** on the [decision model](#decision-model) or on the Anthropic key under [Code review](#code-review-2) and paste the new key. Triage restarts its workers for every host. +**Model keys.** Press **Change** on the [decision model](#decision-model) or on the Anthropic key under [Code review](#code-review-2) and paste the new key. ### Code review @@ -629,7 +627,7 @@ Type in the search box to narrow the list by actor, action or pull request. The | Role | A workspace role (Owner, Admin, Member) that controls what a person may do in Triage. | | CI | Continuous integration: the automated tests and checks a repository runs on every pull request. | | Draft | A pull request its author has marked as not ready. Triage waits until it is marked ready. | -| Code review | Triage's optional bug finder, switched on per repository or by its policy file. It reads each new commit and lists the bugs it finds in Triage, and on GitHub sends them to the pull request when you ask. | +| Code review | Triage's optional bug finder, turned on by a repository's policy file. It reads each new commit and lists the bugs it finds in Triage, and on GitHub sends them to the pull request when you ask. | | Finding | One bug code review reports, with its severity (P0 is the most severe), file and line. | | First-timer | Someone opening their first pull request in a repository. | | Webhook | The channel GitHub or Bitbucket uses to tell Triage about new events. | diff --git a/docs/Corbits Triage User Manual/images/27.png b/docs/Corbits Triage User Manual/images/27.png deleted file mode 100644 index 427151f98..000000000 Binary files a/docs/Corbits Triage User Manual/images/27.png and /dev/null differ diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index c06b0235a..b1513660d 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -27,7 +27,7 @@ with the `intx-src` export condition, so every hub command runs with | `src/audit-log.ts` | The audit list, read from what the workspace already recorded. | | `src/invitations/` | Invites as stock invited principals with magic links, and invite email over SMTP. | | `src/workflow-deploy.ts`, `src/workflow-versions.ts` | Deploys the portal's published workflow versions; lists what is deployed. | -| `src/code-review.ts`, `src/code-review-feed.ts`, `src/code-review-runs.ts` | The code review switch; `…/place`, the selected provisioner and each place probed at boot (`dockerPlaces`); `…/run`; the feed that mails each new head; the latest review, read from the manager's state, and whether a head takes it over. | +| `src/code-review.ts`, `src/pull-request-reader.ts`, `src/code-review-runs.ts` | The code review switch; `…/place`, the selected provisioner and each place probed at boot (`dockerPlaces`); `…/run`; reading a pull request and building its review request; the latest review, read from the manager's state, and whether a head takes it over. | | `src/sidecar/` | Local-process `SidecarProvisioner` (lifecycle, spawn env, encrypted manifests, boot restore, `host-capacity.ts`) and Docker registration (`docker.ts`). | | `src/sidecar-config.ts` | Sidecar adapter manifest (System One model provider). | | `src/bitbucket-datacenter/` | Bitbucket Data Center connection, repository and webhook routes, payload normalizing. | @@ -36,8 +36,7 @@ with the `intx-src` export condition, so every hub command runs with | `src/github/policy-file.ts`, `check-pack-store.ts`, `policy-read.ts`, `policy-pull.ts` | The policy file (read again when a push touches it or a pull request merges into the default branch, and at most a minute old while the repository's webhook (the App's, on GitHub) has been silent for half an hour), the stored policy and which applies, the policy route, and the one `corbits-triage/policy` pull request. | | `src/github/automation.ts` | Which action kinds triage writes unattended, from the `pr-triage` role's grants. | | `src/github/coalescer.ts` | Folds a pull request's webhook burst into one mail. | -| `src/github/coordinator.ts`, `worker-spawn.ts` | The job queue and one worker deployment per job. | -| `src/github/triage-reconciler.ts`, `reconcile-plan.ts`, `reconcile-loop.ts`, `triage-queue.ts` | Re-queues untriaged and stale heads, rotates deployments and redeploys failed ones. | +| `src/github/triage-reconciler.ts`, `reconcile-plan.ts`, `reconcile-loop.ts`, `triage-queue.ts` | Re-queues untriaged and stale heads through the listener. | | `src/github/triage-state-store.ts` | The per-repository triage state artifact. | | `src/github/tenant-observer.ts`, `run-recorder.ts`, `triage-runs.ts` | Reads run logs and records verdicts. | | `src/github/verdict-record-store.ts`, `triage-verdicts.ts`, `triage-stats.ts`, `triage-timeline.ts`, `pr-outcomes.ts` | Verdict records and the routes that serve them, including outcomes before and after Triage. Outcomes read at most the 50 newest-created pull requests on GitHub, or the newest page holding any (at most 8 pages of 50) on Bitbucket Data Center, with at most 8 reads at once, cached per pull request and per UTC day. | @@ -50,19 +49,14 @@ with the `intx-src` export condition, so every hub command runs with ## Portal (`apps/web`) -- `src/lib/portal.tsx`: convergence on sign-in. +- `src/lib/portal.tsx`: the workspace snapshot and the Settings actions. - `src/lib/hub-api.ts`: typed calls over the hub routes. - `src/lib/model-catalog.ts`: the models the workflows name (`decision` on `corbits-system-one`, `claude-opus-5-5` and `claude-haiku-5-5` on `anthropic`); adds what the tenant catalog lacks through the stock catalog routes, and picks each deploy's offerings. -- `src/lib/workflow-deploy.ts`: publishes the tool and workflow tarballs to the - tenant's `corbits` package registry and deploys each workflow pinned to its - version, redeploying only when the version or its model's key changed. Code - review deploys only while a connected repository has it on and a provider - key is stored. Before publishing it removes tarballs neither this build nor - a live deployment pins, since Interchange delivers the registry's whole tree - under a 32 MiB cap. +- `src/lib/workflow-deploy.ts`: workflow assets, the build's files, the + tenant's deployment lifetime and cancelling a deployment. - `src/lib/workflow-sources.ts`: publishes each workflow as a source-format `workflow` asset, through the stock git smart-HTTP route with a short-lived personal git token that is revoked afterwards, and returns each asset id and @@ -99,10 +93,8 @@ with the `intx-src` export condition, so every hub command runs with The workflows name one decision model, `corbits-system-one:decision`, so their source is the same for every tenant. Saving a model key stores it in the vault and writes the `decision` offering (endpoint as the provider's `baseURL`, the -provider's own model as `quirks.model`), then redeploys, because a deployment -resolves its offering when deployed. Removing a key first releases the -workflows that use it, then deletes its catalog provider and offerings, since -the vault keeps a key a catalog provider uses. +provider's own model as `quirks.model`). Removing a key deletes its catalog +provider and offerings, since the vault keeps a key a catalog provider uses. The portal reads two build-time variables: `VITE_HUB_URL` (the hub's origin when the portal is hosted apart) and `VITE_GITHUB_API_ORIGIN` (the GitHub diff --git a/packages/triage-contracts/src/hub-responses.ts b/packages/triage-contracts/src/hub-responses.ts index 0e972c5f4..94ae53040 100644 --- a/packages/triage-contracts/src/hub-responses.ts +++ b/packages/triage-contracts/src/hub-responses.ts @@ -206,6 +206,8 @@ export const DeployedVersions = type({ worker: "boolean", package: PackageVersion.or("null"), tools: PackageVersion.array(), + /** The source asset commit a source-format deployment runs, so it can be deployed again as it is. */ + source: type({ assetId: "string", commitSha: "string" }).or("null"), }).array(), }); export type DeployedVersions = typeof DeployedVersions.infer;