diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 51b6889ad..e26efe018 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -16,6 +16,7 @@ import { createSidecarAllocationStore, createSignalCorrelationStore, createWorkflowRunDispatchStore, + listAssetsForTenant, listVisibleOfferings, resolveCredentialByName, resolveCredentialRequirement, @@ -360,7 +361,6 @@ import { type Context, Hono, type Next } from "hono"; import { upgradeWebSocket, websocket } from "hono/bun"; import { CORBITS_TOOLS_REGISTRY, - describeCorbitsToolPackages, publishCorbitsToolsRegistry, } from "@corbits/tool-registry-publish"; import { @@ -834,20 +834,6 @@ export async function createHub(config: HubConfig) { }, }; const hubPublicKey = hexEncode(signingKey.publicKey); - // CL-6149: a launch's pinned tool packages (`toolPackagePins`) carry - // no grants of their own — the deploy-time capability walk - // (`vendor/intx/workflow-deploy/src/capability-walk.ts`) only derives - // `tool:` grants for inline tool factories, so a pinned package's - // tools failed every call closed with "No matching grants". Every - // `@corbits/*-tools` package's namespaced tool ids and approval marks - // are read once here (`describeCorbitsToolPackages`), so - // `toolGrantsForPins` — the port `createHubChatPlatform`'s - // `CreateHubChatPlatformDeps` is built with — can synchronously turn a - // launch's pins into the `tool:` grants minted against - // the run's own principal. - const toolGrantsForPins = createToolGrantsForPins( - await describeCorbitsToolPackages(), - ); // Same owning check GET /connections uses — see // `@corbits/connections`' `workflow-connection-routes.ts` and the // `createWorkflowConnectionRoutes` wiring below. Not @@ -1064,6 +1050,18 @@ export async function createHub(config: HubConfig) { assetService, repoStore: agentRepoStore.repoStore, }); + // CL-6149: a launch's pinned tool packages (`toolPackagePins`) carry + // no grants of their own — the deploy-time capability walk + // (`vendor/intx/workflow-deploy/src/capability-walk.ts`) only derives + // `tool:` grants for inline tool factories. `toolGrantsForPins` — the + // port `createHubChatPlatform`'s `CreateHubChatPlatformDeps` is built + // with — turns a launch's pins into `tool:` grants read + // from the tenant-resolved `corbits-tools` asset's packed manifests + // (CL-7582), through the same cached launch-path read seam above. + const toolGrantsForPins = createToolGrantsForPins({ + listAssets: (tenantId, kind) => listAssetsForTenant(db, tenantId, kind), + assetService: launchCaches.assetService, + }); const launchAgentRepoStore: AgentRepoStore = { writeDeployTree: agentRepoStore.writeDeployTree, createDeployPack: agentRepoStore.createDeployPack, diff --git a/apps/hub/src/tool-grants.test.ts b/apps/hub/src/tool-grants.test.ts index 13dacd1e2..6dcfabe32 100644 --- a/apps/hub/src/tool-grants.test.ts +++ b/apps/hub/src/tool-grants.test.ts @@ -1,109 +1,165 @@ // CL-6149: proves the hub's `toolGrantsForPins` port turns a launch's // `toolPackagePins` into the exact `tool:` grants the -// workflow child's authz gate matches against. +// workflow child's authz gate matches against — derived, since CL-7582, +// from the installed `corbits-tools` asset's packed manifests rather +// than any source import. import { describe, expect, test } from "bun:test"; -import { describeCorbitsToolPackages } from "@corbits/tool-registry-publish"; +import type { AssetWithOrigin } from "@intx/db"; +import type { + AssetService, + ListAssetBlobsParams, + ReadAssetBlobParams, +} from "@intx/hub-sessions"; +import { + packToolPackageTarball, + tarballFilenameFor, +} from "@corbits/tool-registry-publish"; import { createToolGrantsForPins } from "./tool-grants"; -const DESCRIPTIONS = [ - { - name: "@corbits/memory-tools", - version: "0.0.1", - tools: [ - { - qualifiedId: "@corbits/memory-tools/memory:memory_add", - approval: "ask" as const, - }, - { - qualifiedId: "@corbits/memory-tools/memory:memory_list", - }, - ], - }, - { - name: "@corbits/connections-tools", - version: "0.0.1", - tools: [ - { - qualifiedId: "@corbits/connections-tools/connections:list_connections", - }, - ], - }, -]; +async function fakeRegistryAssetService( + tarballs: Awaited>[], +): Promise> { + const blobs = new Map( + tarballs.map((tarball) => [ + `tarballs/${tarballFilenameFor(tarball.name, tarball.version)}`, + tarball.bytes, + ]), + ); + return { + listAssetBlobs: (params: ListAssetBlobsParams) => + Promise.resolve( + [...blobs.keys()].filter((path) => path.startsWith(`${params.dir}/`)), + ), + readAssetBlob: (params: ReadAssetBlobParams) => { + const bytes = blobs.get(params.path); + if (bytes === undefined) { + return Promise.reject(new Error(`no ${params.path}`)); + } + return Promise.resolve(bytes); + }, + }; +} + +function assetRow(id: string, direct: boolean): AssetWithOrigin { + return { + id, + name: "corbits-tools", + origin: { tenantId: "tenant_root", direct }, + } as AssetWithOrigin; +} + +const MEMORY_DIR = new URL("../../../packages/memory-tools", import.meta.url) + .pathname; +const MANUS_DIR = new URL("../../../packages/manus-tools", import.meta.url) + .pathname; + +async function grantsFor( + assetService: Pick, + assets: readonly AssetWithOrigin[], + pins: readonly { name: string; version: string }[], +) { + return createToolGrantsForPins({ + listAssets: (tenantId, kind) => { + expect(tenantId).toBe("tenant_child"); + expect(kind).toBe("package-registry"); + return Promise.resolve(assets); + }, + assetService, + })("tenant_child", pins); +} describe("createToolGrantsForPins", () => { - test("mints tool:/invoke for every tool a pinned package declares", () => { - const toolGrantsForPins = createToolGrantsForPins(DESCRIPTIONS); - const grants = toolGrantsForPins([ - { name: "@corbits/memory-tools", version: "^1" }, + test("mints tool:/invoke for every tool the installed manifest declares", async () => { + const assetService = await fakeRegistryAssetService([ + await packToolPackageTarball(MEMORY_DIR), ]); - expect(grants).toEqual([ - { - resource: "tool:@corbits/memory-tools/memory:memory_add", - action: "invoke", - effect: "ask", - }, - { - resource: "tool:@corbits/memory-tools/memory:memory_list", - action: "invoke", - effect: "allow", - }, + const grants = await grantsFor( + assetService, + [assetRow("asset_1", true)], + [{ name: "@corbits/memory-tools", version: "^1" }], + ); + expect(grants.map((grant) => grant.resource)).toEqual([ + "tool:@corbits/memory-tools/memory:memory_search", + "tool:@corbits/memory-tools/memory:memory_add", + "tool:@corbits/memory-tools/memory:memory_list", ]); + expect(grants.every((grant) => grant.action === "invoke")).toBe(true); }); - test('floors an unmarked tool at allow and a `approval: "ask"` tool at ask', () => { - const toolGrantsForPins = createToolGrantsForPins(DESCRIPTIONS); - const grants = toolGrantsForPins([ - { name: "@corbits/memory-tools", version: "^1" }, + test('floors an unmarked tool at allow and a `approval: "ask"` tool at ask', async () => { + const assetService = await fakeRegistryAssetService([ + await packToolPackageTarball(MANUS_DIR), ]); - expect(grants.find((g) => g.resource.endsWith("memory_add"))?.effect).toBe( - "ask", - ); - expect(grants.find((g) => g.resource.endsWith("memory_list"))?.effect).toBe( - "allow", + const grants = await grantsFor( + assetService, + [assetRow("asset_1", true)], + [{ name: "@corbits/manus-tools", version: "*" }], ); + expect( + grants.find((grant) => grant.resource.endsWith(":webhook_create")) + ?.effect, + ).toBe("ask"); + expect( + grants.find((grant) => grant.resource.endsWith(":create_slides"))?.effect, + ).toBe("allow"); + expect( + grants.find((grant) => grant.resource.endsWith(":task_list"))?.effect, + ).toBe("allow"); }); - test("unions grants across every pinned package", () => { - const toolGrantsForPins = createToolGrantsForPins(DESCRIPTIONS); - const grants = toolGrantsForPins([ - { name: "@corbits/memory-tools", version: "^1" }, - { name: "@corbits/connections-tools", version: "^1" }, - ]); - expect(grants.map((g) => g.resource)).toEqual([ - "tool:@corbits/memory-tools/memory:memory_add", - "tool:@corbits/memory-tools/memory:memory_list", - "tool:@corbits/connections-tools/connections:list_connections", + test("unions grants across every pinned package and skips unknown pins", async () => { + const assetService = await fakeRegistryAssetService([ + await packToolPackageTarball(MEMORY_DIR), + await packToolPackageTarball(MANUS_DIR), ]); + const grants = await grantsFor( + assetService, + [assetRow("asset_1", true)], + [ + { name: "@corbits/memory-tools", version: "^1" }, + { name: "@corbits/manus-tools", version: "*" }, + { name: "@corbits/unknown-tools", version: "^1" }, + ], + ); + expect( + grants.some((grant) => grant.resource.includes("memory-tools")), + ).toBe(true); + expect(grants.some((grant) => grant.resource.includes("manus-tools"))).toBe( + true, + ); + expect(grants.some((grant) => grant.resource.includes("unknown"))).toBe( + false, + ); }); - test("a pin naming a package the hub does not describe yields no grants, never throws", () => { - const toolGrantsForPins = createToolGrantsForPins(DESCRIPTIONS); - const grants = toolGrantsForPins([ - { name: "@corbits/unknown-tools", version: "^1" }, + test("resolves an inherited registry, not only a direct one", async () => { + const assetService = await fakeRegistryAssetService([ + await packToolPackageTarball(MEMORY_DIR), ]); - expect(grants).toEqual([]); + const grants = await grantsFor( + assetService, + [assetRow("asset_root", false)], + [{ name: "@corbits/memory-tools", version: "^1" }], + ); + expect(grants.length).toBeGreaterThan(0); }); - test("no pins yields no grants", () => { - const toolGrantsForPins = createToolGrantsForPins(DESCRIPTIONS); - expect(toolGrantsForPins([])).toEqual([]); + test("a tenant with no corbits-tools asset yields no grants, never throws", async () => { + const assetService = await fakeRegistryAssetService([]); + const grants = await grantsFor( + assetService, + [], + [{ name: "@corbits/memory-tools", version: "^1" }], + ); + expect(grants).toEqual([]); }); - test("assistant pin of webhook_create is ask, not allow", async () => { - const toolGrantsForPins = createToolGrantsForPins( - await describeCorbitsToolPackages(), - ); - const grants = toolGrantsForPins([ - { name: "@corbits/manus-tools", version: "*" }, + test("no pins yields no grants", async () => { + const assetService = await fakeRegistryAssetService([ + await packToolPackageTarball(MEMORY_DIR), ]); expect( - grants.find((g) => g.resource.endsWith(":webhook_create"))?.effect, - ).toBe("ask"); - expect( - grants.find((g) => g.resource.endsWith(":create_slides"))?.effect, - ).toBe("allow"); - expect(grants.find((g) => g.resource.endsWith(":task_list"))?.effect).toBe( - "allow", - ); + await grantsFor(assetService, [assetRow("asset_1", true)], []), + ).toEqual([]); }); }); diff --git a/apps/hub/src/tool-grants.ts b/apps/hub/src/tool-grants.ts index 6358dd365..25de72cf6 100644 --- a/apps/hub/src/tool-grants.ts +++ b/apps/hub/src/tool-grants.ts @@ -2,26 +2,62 @@ // grants of their own — the deploy-time capability walk // (`vendor/intx/workflow-deploy/src/capability-walk.ts`) only derives // `tool:` grants for inline tool factories, so a pinned package's tools -// failed every call closed with "No matching grants". This builds the -// `ToolGrantsForPins` port `createHubChatPlatform`'s -// `CreateHubChatPlatformDeps` is wired with: given a launch's pins, look -// up each pin's package by name -// in the hub's own `describeCorbitsToolPackages()` read and mint one -// `tool:` / `invoke` declaration per tool, floored at `ask` -// for a tool the package itself marks `approval: "ask"`. +// failed every call closed with "No matching grants". Given a launch's +// pins, this reads the tenant-resolved `corbits-tools` package-registry +// asset's packed tarballs through `readToolSurfaceManifests` and mints +// one `tool:` / `invoke` declaration per tool, floored at +// `ask` for a tool the package itself marks `approval: "ask"` — the +// grants come from the installed asset's own manifest, never from a +// source import. import type { PinnedToolGrantDeclaration, ToolGrantsForPins, } from "@corbits/chat"; -import type { CorbitsToolPackageDescription } from "@corbits/tool-registry-publish"; +import { + CORBITS_TOOLS_REGISTRY, + readToolSurfaceManifests, + type ToolSurfaceBlobSource, +} from "@corbits/tool-registry-publish"; +import type { AssetWithOrigin } from "@intx/db"; +import type { AssetService } from "@intx/hub-sessions"; + +export type CreateToolGrantsForPinsDeps = { + /** + * Resolves the `package-registry` assets a tenant sees, local first, + * shadows-wins down the ancestor chain. The hub binds + * `@intx/db`'s `listAssetsForTenant` to its own db handle. + */ + listAssets: ( + tenantId: string, + kind: string, + ) => Promise; + /** The launch-path asset service — the launch caches' SHA-keyed wrapper. */ + assetService: Pick; +}; export function createToolGrantsForPins( - descriptions: readonly CorbitsToolPackageDescription[], + deps: CreateToolGrantsForPinsDeps, ): ToolGrantsForPins { - const toolsByPackageName = new Map( - descriptions.map((description) => [description.name, description.tools]), - ); - return (pins) => { + return async (tenantId, pins) => { + const assets = await deps.listAssets(tenantId, "package-registry"); + const registry = assets.find((row) => row.name === CORBITS_TOOLS_REGISTRY); + if (registry === undefined) return []; + + const source: ToolSurfaceBlobSource = { + listBlobs: (dir) => + deps.assetService.listAssetBlobs({ assetId: registry.id, dir }), + readBlob: (path) => + deps.assetService.readAssetBlob({ assetId: registry.id, path }), + rootDir: "tarballs", + }; + const manifests = await readToolSurfaceManifests(source); + const toolsByPackageName = new Map( + manifests.map((manifest) => [ + manifest.name, + manifest.surface.filter((entry) => entry.kind === "tool"), + ]), + ); + const grants: PinnedToolGrantDeclaration[] = []; for (const pin of pins) { const tools = toolsByPackageName.get(pin.name); diff --git a/docs/local-dev.md b/docs/local-dev.md index fd2a775e8..06d4be7e7 100644 --- a/docs/local-dev.md +++ b/docs/local-dev.md @@ -39,8 +39,12 @@ A workflow that pins a `@corbits/*` tool package (e.g. **assistant** pinning tenant when onboarding or an explicit `@corbits/seeding` caller asks; descendants inherit it, and `seedTenant` does not pack. After changing a tool package's source, bump its version, then publish onto the tenant that -owns the registry — restarting the hub does not republish. -Resolution and the sidecar's materialized store key on +owns the registry — restarting the hub does not republish. The operator +path is `bun run publish-tools` (with `HUB_ADMIN_EMAIL`/`HUB_ADMIN_PASSWORD` +set, and `--tenant ` when the admin belongs to more than one +tenant): it signs in, resolves the target tenant, and installs the +registry onto that already-existing tenant over the hub's native asset +routes. Resolution and the sidecar's materialized store key on `name@version`, not on content, so republishing unchanged-version bytes never reaches a running or freshly-launched agent; `tool-registry-publish` refuses to overwrite an existing `name@version` with different content for diff --git a/docs/seed-reconciliation.md b/docs/seed-reconciliation.md index 0344df3b5..f0f37c750 100644 --- a/docs/seed-reconciliation.md +++ b/docs/seed-reconciliation.md @@ -136,6 +136,13 @@ finds-or-creates the tenant's `corbits-tools` package-registry asset, then PUTs whatever tarball is missing. Onboarding and explicit `@corbits/seeding` callers publish onto a tenant so descendants inherit tarballs; `seedTenant` itself does not pack. Hub boot does not publish. +The operator-facing install path is `bun run publish-tools` +(`scripts/publish-tools.ts`), which signs an existing admin in and +publishes onto an already-existing tenant — the same find-or-create, +409-tolerant asset flow, never a new provisioning path. Each packed +tarball's `package.json` also carries the `ToolSurfaceManifest` the +hub reads back (`readToolSurfaceManifests`) to mint pinned-tool grants, +so grants always describe the bytes actually installed on the tenant. Two properties keep a failed publish from stranding a usable-looking-but-empty asset: diff --git a/package.json b/package.json index c0e98fa17..12257cd3b 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "test": "bun test ./scripts/*.test.ts && bun run scripts/run-all.ts test", "test:e2e": "bun test scripts/e2e --max-concurrency=1", "dev": "bun run scripts/dev.ts", + "publish-tools": "bun scripts/publish-tools.ts", "setup": "bun scripts/db-setup.ts", "setup:memory": "bun run scripts/setup-memory.ts", "reset": "bun scripts/reset.ts", diff --git a/packages/agent-directory/src/agent-workflow.test.ts b/packages/agent-directory/src/agent-workflow.test.ts index 4cfb4d517..247e8c9dd 100644 --- a/packages/agent-directory/src/agent-workflow.test.ts +++ b/packages/agent-directory/src/agent-workflow.test.ts @@ -4,9 +4,13 @@ // a definition pinning skills carried a pin the corbits-tools registry // could never resolve at launch. import { describe, expect, test } from "bun:test"; +import path from "node:path"; import type { DB } from "@intx/db"; import type { AssetService } from "@intx/hub-sessions"; -import { describeCorbitsToolPackages } from "@corbits/tool-registry-publish"; +import { + CORBITS_TOOL_PACKAGE_DIRS, + packToolPackageTarball, +} from "@corbits/tool-registry-publish"; import { buildAgentDefinitionWorkflow, createAgentDefinitionCore, @@ -18,12 +22,16 @@ import { createInMemoryDefinitionSkillsStore } from "./skills-store"; describe("SKILLS_TOOL_PACKAGE_PIN", () => { test("resolves through the corbits-tools registry", async () => { - const descriptions = await describeCorbitsToolPackages(); - const match = descriptions.find( - (description) => description.name === SKILLS_TOOL_PACKAGE_PIN.name, + // Assert against what the registry actually carries: the packed + // tarball for the pinned package, not a source-tree import. + const dir = CORBITS_TOOL_PACKAGE_DIRS.find( + (candidate) => + path.basename(candidate) === SKILLS_TOOL_PACKAGE_PIN.name.split("/")[1], ); - expect(match).toBeDefined(); - expect(match?.version).toBe(SKILLS_TOOL_PACKAGE_PIN.version); + expect(dir).toBeDefined(); + const tarball = await packToolPackageTarball(dir as string); + expect(tarball.name).toBe(SKILLS_TOOL_PACKAGE_PIN.name); + expect(tarball.version).toBe(SKILLS_TOOL_PACKAGE_PIN.version); }); }); diff --git a/packages/chat/src/pin-ports.ts b/packages/chat/src/pin-ports.ts index 1f56f10f1..bf7fe7da2 100644 --- a/packages/chat/src/pin-ports.ts +++ b/packages/chat/src/pin-ports.ts @@ -11,8 +11,9 @@ export type PinnedToolGrantDeclaration = { }; export type ToolGrantsForPins = ( + tenantId: string, pins: readonly ToolPackagePin[], -) => readonly PinnedToolGrantDeclaration[]; +) => Promise; export type McpCredentialBindingsFor = ( tenantId: string, diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 73d418937..4e7215c8d 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -905,7 +905,7 @@ describe("createHubChatPlatform", () => { const eventCollectors = createFakeEventCollectors(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -984,7 +984,7 @@ describe("createHubChatPlatform", () => { const eventCollectors = createFakeEventCollectors(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -1050,7 +1050,7 @@ describe("createHubChatPlatform", () => { const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -1128,7 +1128,7 @@ describe("createHubChatPlatform", () => { const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -1199,7 +1199,7 @@ describe("createHubChatPlatform", () => { }; const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter: createFakeSidecarRouter(), @@ -1267,7 +1267,7 @@ describe("createHubChatPlatform", () => { const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -1327,7 +1327,7 @@ describe("createHubChatPlatform", () => { const eventCollectors = createFakeEventCollectors(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -1397,7 +1397,7 @@ describe("createHubChatPlatform", () => { }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1465,7 +1465,7 @@ describe("createHubChatPlatform", () => { }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1489,7 +1489,7 @@ describe("createHubChatPlatform", () => { test("refuses to mint the platform when credentialCipher is missing", () => { expect(() => createHubChatPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: {} as never, runTrigger: {} as never, sidecarRouter: {} as never, @@ -1502,7 +1502,7 @@ describe("createHubChatPlatform", () => { test("refuses to mint the platform when credentialCipher has the wrong shape", () => { expect(() => createHubChatPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: {} as never, runTrigger: {} as never, sidecarRouter: {} as never, @@ -1520,7 +1520,7 @@ describe("createHubChatPlatform", () => { await expect( (async () => { const platform = createHubChatPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: {} as never, runTrigger: {} as never, sidecarRouter: {} as never, @@ -1558,7 +1558,7 @@ describe("createHubChatPlatform", () => { tenantRow: { id: "ten_1", domain: "ten1.workbench.test" }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), @@ -1610,7 +1610,7 @@ describe("createHubChatPlatform", () => { }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1682,7 +1682,7 @@ describe("createHubChatPlatform", () => { }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1750,7 +1750,7 @@ describe("createHubChatPlatform", () => { }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1779,7 +1779,7 @@ describe("createHubChatPlatform", () => { tenantRow: { id: "ten_1", domain: "ten1.workbench.test" }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), @@ -1823,7 +1823,7 @@ describe("createHubChatPlatform", () => { }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1872,7 +1872,7 @@ describe("createHubChatPlatform", () => { wireProjectionsByDefinitionId: { wfd_echo: NO_MODEL_PROJECTION }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1914,7 +1914,7 @@ describe("createHubChatPlatform", () => { wireProjectionsByDefinitionId: { wfd_echo: NO_MODEL_PROJECTION }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -1965,7 +1965,7 @@ describe("createHubChatPlatform", () => { ], }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), @@ -1997,7 +1997,7 @@ describe("createHubChatPlatform", () => { const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -2114,7 +2114,7 @@ describe("createHubChatPlatform", () => { const eventCollectors = createFakeEventCollectors(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -2203,7 +2203,7 @@ describe("createHubChatPlatform", () => { const eventCollectors = createFakeEventCollectors(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -2268,7 +2268,7 @@ describe("createHubChatPlatform", () => { }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter, @@ -2332,7 +2332,7 @@ describe("createHubChatPlatform", () => { }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter, @@ -2398,7 +2398,7 @@ describe("createHubChatPlatform", () => { }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter, @@ -2441,7 +2441,7 @@ describe("createHubChatPlatform", () => { definitionId: "wfd_workbench1", }); createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), @@ -2474,7 +2474,7 @@ describe("createHubChatPlatform", () => { definitionId: "wfd_workbench1", }); createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), @@ -2520,7 +2520,7 @@ describe("createHubChatPlatform", () => { }); const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter: createFakeSidecarRouter({ @@ -2589,7 +2589,7 @@ describe("createHubChatPlatform", () => { const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -2651,7 +2651,7 @@ describe("createHubChatPlatform", () => { const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -2674,7 +2674,7 @@ describe("createHubChatPlatform", () => { definitionId: "wfd_workbench1", }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -2811,7 +2811,7 @@ describe("createHubChatPlatform", () => { }; const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter, @@ -2902,7 +2902,7 @@ describe("createHubChatPlatform", () => { test("recomputes and persists the folded body from the definition's current projection", async () => { const db = buildRefreshableDb(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), @@ -2952,7 +2952,7 @@ describe("createHubChatPlatform", () => { }); const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), @@ -3049,7 +3049,7 @@ describe("createHubChatPlatform", () => { }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), @@ -3174,7 +3174,7 @@ describe("createHubChatPlatform stale-definition reconciliation", () => { routableAddresses: opts.routable ? ["run_stale@ten1.workbench.test"] : [], }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter, @@ -3252,7 +3252,7 @@ describe("createHubChatPlatform stale-definition reconciliation", () => { }); const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, sidecarRouter: createFakeSidecarRouter({ @@ -3339,7 +3339,7 @@ describe("createHubChatPlatform stale-definition reconciliation", () => { }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ @@ -3404,7 +3404,7 @@ describe("createHubChatPlatform relaunch sweep", () => { const runTrigger = createFakeRunTrigger(); const notices: unknown[] = []; const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger, // Routable, and dead anyway: that combination is exactly what the @@ -3616,7 +3616,7 @@ describe("createHubChatPlatform inference-source rotation reconciliation", () => }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ @@ -3845,7 +3845,7 @@ describe("createHubChatPlatform pinned-tool-package connect reconciliation", () }, }); const platform = createPlatform({ - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], db: db as never, runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ diff --git a/packages/chat/test/relaunch-close.test.ts b/packages/chat/test/relaunch-close.test.ts index b4a81f59f..d7bff0669 100644 --- a/packages/chat/test/relaunch-close.test.ts +++ b/packages/chat/test/relaunch-close.test.ts @@ -127,7 +127,7 @@ function createFakeDb(opts: { function createPlatform(db: never) { return createHubChatPlatform({ db, - toolGrantsForPins: () => [], + toolGrantsForPins: async () => [], runTrigger: {} as never, repoStore: { resolveRef: async () => "sha_test" }, sidecarRouter: { getRoutableAddresses: () => [] } as never, diff --git a/packages/tool-registry-publish/README.md b/packages/tool-registry-publish/README.md index 6efc9ebc0..59681160f 100644 --- a/packages/tool-registry-publish/README.md +++ b/packages/tool-registry-publish/README.md @@ -24,9 +24,19 @@ for how a pin resolves through it). `package.json`, bundles its `"."` export with the `bun build` CLI (an isolated subprocess — an in-process `Bun.build()` call was observed to fail nondeterministically alongside other live `bun` - processes), and tars the result. Memoized per directory for a - process's lifetime, so concurrent or repeated calls never race two - bundler invocations against the same input. + processes), enumerates the bundled tool factories into a + `ToolSurfaceManifest` (`qualifiedId`/`kind`/optional `ask` approval, + duplicate-qualifiedId-rejecting, arktype-parsed), and tars the result + with that manifest written into the synthesized `package.json`. + Memoized per directory for a process's lifetime, so concurrent or + repeated calls never race two bundler invocations against the same + input. +- `readToolSurfaceManifests` — reads those manifests back out of a + packed `tarballs/*.tgz` blob tree with injected `listBlobs`/`readBlob` + (the hub wires them to its launch-path `assetService` wrapper). The + hub's pinned-tool grants derive from this installed manifest — the + source-importing describer this replaces is gone, so the hub never + imports tool-package sources to grant. - `publishCorbitsToolsRegistry` — find-or-create the tenant's `corbits-tools` asset (409-tolerant, so two overlapping publish runs for the same tenant never both fail on the asset's own name diff --git a/packages/tool-registry-publish/src/describe.test.ts b/packages/tool-registry-publish/src/describe.test.ts deleted file mode 100644 index e4e4aafa5..000000000 --- a/packages/tool-registry-publish/src/describe.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -// `describeCorbitsToolPackages` is the hub-side source of truth for -// which `tool:` grants a pinned `@corbits/*-tools` package -// needs (see CL-6149: a pinned package's tools failed every call with -// "No matching grants" because nothing derived those grants at launch). -// This suite pins the exact qualified-id shape and approval marks the -// hub composition (`apps/hub/src/index.ts`) relies on. -import { describe, expect, test } from "bun:test"; -import { CORBITS_TOOL_PACKAGE_DIRS } from "./registry"; -import { describeCorbitsToolPackages } from "./describe"; - -describe("describeCorbitsToolPackages", () => { - test("describes every registered package with at least one tool", async () => { - const descriptions = await describeCorbitsToolPackages(); - expect(descriptions.length).toBe(CORBITS_TOOL_PACKAGE_DIRS.length); - for (const description of descriptions) { - expect(description.name.startsWith("@corbits/")).toBe(true); - expect(description.version.length).toBeGreaterThan(0); - expect(description.tools.length).toBeGreaterThan(0); - } - }); - - test("every tool's qualifiedId is `:`, matching the loader's namespace prefix", async () => { - const descriptions = await describeCorbitsToolPackages(); - for (const description of descriptions) { - for (const tool of description.tools) { - expect(tool.qualifiedId).toMatch(/^.+:[^:]+$/); - } - } - }); - - test("caches across calls (same source for the process lifetime)", async () => { - const first = await describeCorbitsToolPackages(); - const second = await describeCorbitsToolPackages(); - expect(second).toBe(first); - }); - - test("tools-skills grants exactly one qualifiedId for skills_load, not the old load_skill name", async () => { - const descriptions = await describeCorbitsToolPackages(); - const skills = descriptions.find( - (description) => description.name === "@corbits/tools-skills", - ); - expect(skills).toBeDefined(); - const qualifiedIds = skills?.tools.map((tool) => tool.qualifiedId) ?? []; - expect(qualifiedIds).toContain("@corbits/tools-skills/skills:skills_load"); - expect(qualifiedIds).not.toContain( - "@corbits/tools-skills/skills:load_skill", - ); - }); -}); diff --git a/packages/tool-registry-publish/src/describe.ts b/packages/tool-registry-publish/src/describe.ts deleted file mode 100644 index 1a51f78ec..000000000 --- a/packages/tool-registry-publish/src/describe.ts +++ /dev/null @@ -1,97 +0,0 @@ -// Static description of every published `@corbits/*-tools` package's -// tool surface, read by importing each package's own `src/index.ts` -// module (the pre-bundle source, not the packed tarball). The hub's -// chat launch composition (`apps/hub/src/index.ts`) uses this to -// derive the `tool:` grants a launch's pinned packages -// need — see `@corbits/chat`'s `ToolGrantsForPins` for why those -// grants have to be minted at deploy time rather than left to the -// deploy-time capability walk, which only covers inline tool factories. -import { readFile } from "node:fs/promises"; -import path from "node:path"; -import { type } from "arktype"; -import { CORBITS_TOOL_PACKAGE_DIRS } from "./registry"; - -const PackageManifest = type({ - name: "string", - version: "string", -}); - -/** Structural shape of an `AnnotatedToolFactory` export, checked by duck type rather than `instanceof` since the loaded module crosses a dynamic `import()` boundary. */ -type Bundle = { - readonly id: string; - readonly definitions: readonly { - readonly name: string; - readonly approval?: "ask"; - }[]; -}; - -function isBundle(value: unknown): value is Bundle { - return ( - typeof value === "function" && - typeof (value as { id?: unknown }).id === "string" && - Array.isArray((value as { definitions?: unknown }).definitions) - ); -} - -export type CorbitsToolPackageTool = { - readonly qualifiedId: string; - readonly approval?: "ask"; -}; - -export type CorbitsToolPackageDescription = { - readonly name: string; - readonly version: string; - readonly tools: readonly CorbitsToolPackageTool[]; -}; - -async function describeOnePackage( - dir: string, -): Promise { - const manifestJson: unknown = JSON.parse( - await readFile(path.join(dir, "package.json"), "utf8"), - ); - const manifest = PackageManifest(manifestJson); - if (manifest instanceof type.errors) { - throw new Error( - `describeCorbitsToolPackages: ${dir}'s package.json failed validation: ${manifest.summary}`, - ); - } - - const mod = (await import(path.join(dir, "src", "index.ts"))) as Record< - string, - unknown - >; - const bundles = Object.values(mod).filter(isBundle); - const tools: CorbitsToolPackageTool[] = []; - for (const bundle of bundles) { - for (const definition of bundle.definitions) { - tools.push({ - qualifiedId: `${bundle.id}:${definition.name}`, - ...(definition.approval !== undefined - ? { approval: definition.approval } - : {}), - }); - } - } - return { name: manifest.name, version: manifest.version, tools }; -} - -// Every corbits tool package's module graph is static for this -// process's lifetime (its source only changes across a restart), so -// the description is computed once and reused — mirrors `pack.ts`'s -// own `packCache` reasoning for the same source directories. -let cached: Promise | undefined; - -/** - * Describe every `@corbits/*-tools` package's exported tool bundles: - * each bundle's namespaced tool id (`:`, - * the exact shape the workflow child's authz gate matches — see - * `@intx/tool-packaging/src/loader.ts`'s `applyNamespacePrefix`) - * and its static approval mark. - */ -export function describeCorbitsToolPackages(): Promise< - readonly CorbitsToolPackageDescription[] -> { - cached ??= Promise.all(CORBITS_TOOL_PACKAGE_DIRS.map(describeOnePackage)); - return cached; -} diff --git a/packages/tool-registry-publish/src/index.ts b/packages/tool-registry-publish/src/index.ts index 7c5647281..19881097d 100644 --- a/packages/tool-registry-publish/src/index.ts +++ b/packages/tool-registry-publish/src/index.ts @@ -5,16 +5,16 @@ export { tarballCoversPackage, tarballsCoverRequiredSeedPackages, } from "./registry"; -export { - describeCorbitsToolPackages, - type CorbitsToolPackageDescription, - type CorbitsToolPackageTool, -} from "./describe"; export { packToolPackageTarball, tarballFilenameFor, type PackedTarball, } from "./pack"; +export { ToolSurfaceEntry, ToolSurfaceManifest } from "./manifest"; +export { + readToolSurfaceManifests, + type ToolSurfaceBlobSource, +} from "./surface-reader"; export { shouldPublishTarball, publishCorbitsToolsRegistry, diff --git a/packages/tool-registry-publish/src/manifest.test.ts b/packages/tool-registry-publish/src/manifest.test.ts new file mode 100644 index 000000000..c6afbb49b --- /dev/null +++ b/packages/tool-registry-publish/src/manifest.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test } from "bun:test"; +import { type } from "arktype"; +import { ToolSurfaceManifest } from "./manifest"; + +describe("ToolSurfaceManifest", () => { + const valid = { + name: "@corbits/memory-tools", + version: "0.0.1", + surface: [ + { + qualifiedId: "@corbits/memory-tools/memory:memory_add", + kind: "tool", + approval: "ask", + }, + { + qualifiedId: "@corbits/memory-tools/memory:memory_list", + kind: "tool", + }, + ], + }; + + test("accepts a real surface", () => { + const manifest = ToolSurfaceManifest(valid); + expect(manifest).not.toBeInstanceOf(type.errors); + if (!(manifest instanceof type.errors)) { + expect(manifest.name).toBe("@corbits/memory-tools"); + expect(manifest.surface).toHaveLength(2); + } + }); + + test("rejects a surface with duplicate qualifiedIds", () => { + const [first, second] = valid.surface; + if (first === undefined || second === undefined) throw new Error("fixture"); + const manifest = ToolSurfaceManifest({ + ...valid, + surface: [first, { ...second, qualifiedId: first.qualifiedId }], + }); + expect(manifest).toBeInstanceOf(type.errors); + }); + + test("rejects malformed entries", () => { + expect( + ToolSurfaceManifest({ + ...valid, + surface: [{ qualifiedId: "x", kind: "widget" }], + }), + ).toBeInstanceOf(type.errors); + expect( + ToolSurfaceManifest({ ...valid, surface: [{ kind: "tool" }] }), + ).toBeInstanceOf(type.errors); + expect(ToolSurfaceManifest({ ...valid, surface: "nope" })).toBeInstanceOf( + type.errors, + ); + }); + + test("accepts a skill-kind entry (manifest headroom)", () => { + const manifest = ToolSurfaceManifest({ + ...valid, + surface: [ + { qualifiedId: "@corbits/skills/s:skills_load", kind: "skill" }, + ], + }); + expect(manifest).not.toBeInstanceOf(type.errors); + }); +}); diff --git a/packages/tool-registry-publish/src/manifest.ts b/packages/tool-registry-publish/src/manifest.ts new file mode 100644 index 000000000..c711dd883 --- /dev/null +++ b/packages/tool-registry-publish/src/manifest.ts @@ -0,0 +1,36 @@ +// The tool-surface manifest a packed `@corbits/*-tools` tarball carries +// in its synthesized `package.json`. The packer derives it from the +// package's own source at pack time; the hub reads it back out of the +// installed `corbits-tools` asset (see `./surface-reader.ts`) to mint +// `tool:` grants — no source imports on the hub side. +import { type } from "arktype"; + +export const ToolSurfaceEntry = type({ + qualifiedId: "string", + kind: "'tool' | 'skill'", + "approval?": "'ask'", +}); +export type ToolSurfaceEntry = typeof ToolSurfaceEntry.infer; + +// A qualifiedId is the binding/delivery key the workflow child's authz +// gate matches grants against, so a duplicate within one manifest is a +// defect the parse boundary must reject rather than collapse silently — +// the same invariant `ToolCredentialDeclarationArray` enforces for +// credential handles. +export const ToolSurfaceManifest = type({ + name: "string", + version: "string", + surface: ToolSurfaceEntry.array().narrow((entries, ctx) => { + const seen = new Set(); + for (const entry of entries) { + if (seen.has(entry.qualifiedId)) { + return ctx.mustBe( + `an array with no duplicate qualifiedIds; "${entry.qualifiedId}" appears more than once`, + ); + } + seen.add(entry.qualifiedId); + } + return true; + }), +}); +export type ToolSurfaceManifest = typeof ToolSurfaceManifest.infer; diff --git a/packages/tool-registry-publish/src/pack.test.ts b/packages/tool-registry-publish/src/pack.test.ts index 9328ecf19..188c847d2 100644 --- a/packages/tool-registry-publish/src/pack.test.ts +++ b/packages/tool-registry-publish/src/pack.test.ts @@ -3,8 +3,10 @@ import * as tar from "tar"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { type } from "arktype"; import { CORBITS_TOOL_PACKAGE_DIRS, CORBITS_TOOLS_REGISTRY } from "./registry"; import { packToolPackageTarball, tarballFilenameFor } from "./pack"; +import { ToolSurfaceManifest } from "./manifest"; // The kind handler's filename rule // (vendor/intx/hub-sessions/src/package-registry-kind.ts @@ -54,23 +56,47 @@ describe("packToolPackageTarball", () => { name: string; version: string; interchange?: { tools?: string }; + surface?: unknown; }; expect(pkgJson.name).toBe(tarball.name); expect(pkgJson.version).toBe(tarball.version); expect(pkgJson.interchange?.tools).toBe("./tool.mjs"); - // The bundle must be import()-able on its own, with no bare - // (non-relative) specifiers left unresolved — exactly what the - // sidecar's tool loader does after extracting the tarball. + // The synthesized package.json must carry a valid tool-surface + // manifest — the hub reads this, not the source tree, for grants. + const surface = ToolSurfaceManifest(pkgJson); + expect(surface).not.toBeInstanceOf(type.errors); + + // Loader parity: the manifest's qualifiedIds must equal exactly + // what the sidecar's tool loader sees when it import()s the + // packed bundle — `:` per definition + // (see `@intx/tool-packaging/src/loader.ts`'s + // `applyNamespacePrefix`). const bundlePath = path.join(extractDir, "package", "tool.mjs"); const mod = (await import(bundlePath)) as Record; const factories = Object.values(mod).filter( (value) => - (typeof value === "function" || typeof value === "object") && - value !== null && - "id" in (value as object), + typeof value === "function" && + typeof (value as { id?: unknown }).id === "string" && + Array.isArray((value as { definitions?: unknown }).definitions), ); expect(factories.length).toBeGreaterThan(0); + if (!(surface instanceof type.errors)) { + const loaderIds = factories.flatMap((factory) => + ( + factory as { + id: string; + definitions: { name: string }[]; + } + ).definitions.map( + (definition) => + `${(factory as { id: string }).id}:${definition.name}`, + ), + ); + expect([...surface.surface.map((e) => e.qualifiedId)].sort()).toEqual( + [...loaderIds].sort(), + ); + } if (tarball.name === "@corbits/catalog-tools") { // Both bundles this package exports (the read-only diff --git a/packages/tool-registry-publish/src/pack.ts b/packages/tool-registry-publish/src/pack.ts index 450ba60ce..1b051b783 100644 --- a/packages/tool-registry-publish/src/pack.ts +++ b/packages/tool-registry-publish/src/pack.ts @@ -18,6 +18,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { type } from "arktype"; import * as tar from "tar"; +import { ToolSurfaceManifest, type ToolSurfaceEntry } from "./manifest"; const BUNDLE_ENTRY_FILENAME = "tool.mjs"; @@ -126,6 +127,57 @@ async function runBunBuild( } } +// The slice of an `AnnotatedToolFactory` export the surface enumeration +// reads, checked by duck type rather than `instanceof` since the loaded +// module crosses a dynamic `import()` boundary — the same shape +// `describe.ts` read before the surface moved into the packed tarball. +type Bundle = { + readonly id: string; + readonly definitions: readonly { + readonly name: string; + readonly approval?: "ask"; + }[]; +}; + +function isBundle(value: unknown): value is Bundle { + return ( + typeof value === "function" && + typeof (value as { id?: unknown }).id === "string" && + Array.isArray((value as { definitions?: unknown }).definitions) + ); +} + +async function surfaceFor( + entryFile: string, + packageName: string, +): Promise { + const mod = (await import(entryFile)) as Record; + const surface: ToolSurfaceEntry[] = []; + for (const value of Object.values(mod)) { + if (!isBundle(value)) continue; + for (const definition of value.definitions) { + surface.push({ + qualifiedId: `${value.id}:${definition.name}`, + kind: "tool", + ...(definition.approval !== undefined + ? { approval: definition.approval } + : {}), + }); + } + } + const manifest = ToolSurfaceManifest({ + name: packageName, + version: "0.0.0", + surface, + }); + if (manifest instanceof type.errors) { + throw new Error( + `packToolPackageTarball: ${packageName}'s tool surface failed validation: ${manifest.summary}`, + ); + } + return manifest.surface; +} + async function packToolPackageTarballUncached( packageDir: string, ): Promise { @@ -139,17 +191,16 @@ async function packToolPackageTarballUncached( ); } + const entryFile = entryFileFor(manifest, packageDir); + const surface = await surfaceFor(entryFile, manifest.name); + const bundleStagingDir = await mkdtemp( path.join(tmpdir(), "corbits-tools-bundle-"), ); let bundleBytes: Uint8Array; try { const outfile = path.join(bundleStagingDir, BUNDLE_ENTRY_FILENAME); - await runBunBuild( - entryFileFor(manifest, packageDir), - outfile, - manifest.name, - ); + await runBunBuild(entryFile, outfile, manifest.name); bundleBytes = new Uint8Array(await readFile(outfile)); } finally { await rm(bundleStagingDir, { recursive: true, force: true }); @@ -159,6 +210,7 @@ async function packToolPackageTarballUncached( name: manifest.name, version: manifest.version, interchange: { tools: `./${BUNDLE_ENTRY_FILENAME}` }, + surface, }; const stagingRoot = await mkdtemp(path.join(tmpdir(), "corbits-tools-pack-")); diff --git a/packages/tool-registry-publish/src/surface-reader.test.ts b/packages/tool-registry-publish/src/surface-reader.test.ts new file mode 100644 index 000000000..1b42cd4a5 --- /dev/null +++ b/packages/tool-registry-publish/src/surface-reader.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import * as tar from "tar"; +import { packToolPackageTarball } from "./pack"; +import { readToolSurfaceManifests } from "./surface-reader"; + +/** A registry tarball published before manifests existed: a real tgz + * whose package.json parses but carries no `surface` key. */ +async function legacyTarball(): Promise { + const dir = await mkdtemp(path.join(tmpdir(), "legacy-pkg-")); + await mkdir(path.join(dir, "package"), { recursive: true }); + await writeFile( + path.join(dir, "package", "package.json"), + JSON.stringify({ + name: "@corbits/pre-manifest-legacy", + version: "1.0.0", + interchange: { tools: {} }, + }), + ); + const out = path.join(dir, "legacy.tgz"); + await tar.create({ cwd: dir, file: out, gzip: true }, ["package"]); + return new Uint8Array(await readFile(out)); +} + +function memorySource(files: Map) { + return { + listBlobs: (dir: string) => + Promise.resolve( + [...files.keys()].filter((path) => path.startsWith(`${dir}/`)), + ), + readBlob: (path: string) => { + const bytes = files.get(path); + if (bytes === undefined) return Promise.reject(new Error(`no ${path}`)); + return Promise.resolve(bytes); + }, + }; +} + +describe("readToolSurfaceManifests", () => { + test("round-trips a packed tarball into a validated manifest", async () => { + const tarball = await packToolPackageTarball( + new URL("../../memory-tools", import.meta.url).pathname, + ); + const source = memorySource( + new Map([[`tarballs/${tarball.filename}`, tarball.bytes]]), + ); + const manifests = await readToolSurfaceManifests({ + ...source, + rootDir: "tarballs", + }); + expect(manifests).toHaveLength(1); + expect(manifests[0]?.name).toBe("@corbits/memory-tools"); + expect(manifests[0]?.surface.map((entry) => entry.qualifiedId)).toContain( + "@corbits/memory-tools/memory:memory_add", + ); + expect(manifests[0]?.surface.every((entry) => entry.kind === "tool")).toBe( + true, + ); + }); + + test("ignores non-tarball blobs and reports none when the registry is empty", async () => { + const source = memorySource( + new Map([["tarballs/README.md", new TextEncoder().encode("hi")]]), + ); + expect( + await readToolSurfaceManifests({ ...source, rootDir: "tarballs" }), + ).toEqual([]); + }); + + test("skips a tarball whose package.json is not a valid manifest", async () => { + const tarball = await packToolPackageTarball( + new URL("../../memory-tools", import.meta.url).pathname, + ); + // Corrupt the packaged manifest by re-packing a tampered file is + // overkill; a truncated tarball is enough to prove the reader skips + // the blob instead of failing the whole read. Registries published + // before manifests existed hold such blobs, and shouldPublishTarball + // never re-uploads an existing name@version to heal them. + const truncated = tarball.bytes.slice(0, 64); + const source = memorySource(new Map([["tarballs/broken.tgz", truncated]])); + expect( + await readToolSurfaceManifests({ ...source, rootDir: "tarballs" }), + ).toEqual([]); + }); + + test("returns the valid manifests and skips the manifest-less legacy ones", async () => { + const tarball = await packToolPackageTarball( + new URL("../../memory-tools", import.meta.url).pathname, + ); + const source = memorySource( + new Map([ + [`tarballs/${tarball.filename}`, tarball.bytes], + ["tarballs/legacy.tgz", await legacyTarball()], + ]), + ); + const manifests = await readToolSurfaceManifests({ + ...source, + rootDir: "tarballs", + }); + expect(manifests).toHaveLength(1); + expect(manifests[0]?.name).toBe("@corbits/memory-tools"); + }); +}); diff --git a/packages/tool-registry-publish/src/surface-reader.ts b/packages/tool-registry-publish/src/surface-reader.ts new file mode 100644 index 000000000..851abe850 --- /dev/null +++ b/packages/tool-registry-publish/src/surface-reader.ts @@ -0,0 +1,86 @@ +// Reads every packed tarball's tool-surface manifest back out of a +// `corbits-tools` package-registry asset's blob tree — the hub-side +// replacement for importing each package's source (`describe.ts`, now +// gone). Blob access is injected so the hub wires it to its launch-path +// `assetService` wrapper (`apps/hub/src/launch-caches.ts`'s SHA-keyed +// `readAssetBlob`), keeping this module free of hub or vendor imports. +import { getLogger } from "@intx/log"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import * as tar from "tar"; +import { type } from "arktype"; +import { ToolSurfaceManifest } from "./manifest"; + +const log = getLogger(["tool-registry-publish", "surface-reader"]); + +export type ToolSurfaceBlobSource = { + /** Lists blob paths under `dir` (repo-root-relative, like `listAssetBlobs`). */ + listBlobs: (dir: string) => Promise; + /** Reads one blob's bytes (repo-root-relative path, like `readAssetBlob`). */ + readBlob: (path: string) => Promise; + /** The asset directory the packed tarballs live under. */ + rootDir: string; +}; + +/** + * Opens each `tarballs/*.tgz` blob, extracts its `package/package.json`, + * and parses it with `ToolSurfaceManifest`. A tarball that carries no + * valid manifest is skipped with a log line, not a thrown error: + * registries published before manifests existed still hold such + * tarballs, and `shouldPublishTarball` never re-uploads an existing + * `name@version` — so one unreadable blob must not take down every + * grants read on a registry that can no longer heal itself. + */ +export async function readToolSurfaceManifests( + source: ToolSurfaceBlobSource, +): Promise { + const blobs = await source.listBlobs(source.rootDir); + const tarballPaths = blobs.filter( + (blob) => blob.startsWith(`${source.rootDir}/`) && blob.endsWith(".tgz"), + ); + const manifests: ToolSurfaceManifest[] = []; + for (const blobPath of tarballPaths) { + const manifest = await manifestFromBlob(blobPath, source); + if (manifest !== undefined) manifests.push(manifest); + } + return manifests; +} + +async function manifestFromBlob( + blobPath: string, + source: ToolSurfaceBlobSource, +): Promise { + let packageJson: unknown; + try { + const bytes = await source.readBlob(blobPath); + packageJson = await extractTarballPackageJSON(bytes); + } catch (error) { + // report-error-ignore: an unreadable tarball in a registry is expected + // legacy data, not an incident — registries published before manifests + // carry such blobs and shouldPublishTarball can never heal them; a log + // line per skip is the whole response. + log.warn`skipping ${blobPath}: unreadable tarball (${error instanceof Error ? error.message : String(error)})`; + return undefined; + } + const manifest = ToolSurfaceManifest(packageJson); + if (manifest instanceof type.errors) { + log.warn`skipping ${blobPath}: package.json is not a valid tool-surface manifest (${manifest.summary})`; + return undefined; + } + return manifest; +} + +async function extractTarballPackageJSON(bytes: Uint8Array): Promise { + const extractDir = await mkdtemp(path.join(tmpdir(), "corbits-surface-")); + try { + const tarballPath = path.join(extractDir, "in.tgz"); + await writeFile(tarballPath, bytes); + await tar.extract({ cwd: extractDir, file: tarballPath }); + return JSON.parse( + await readFile(path.join(extractDir, "package", "package.json"), "utf8"), + ) as unknown; + } finally { + await rm(extractDir, { recursive: true, force: true }); + } +} diff --git a/scripts/e2e/local-rip.test.ts b/scripts/e2e/local-rip.test.ts index 41ee29dd7..1abf76251 100644 --- a/scripts/e2e/local-rip.test.ts +++ b/scripts/e2e/local-rip.test.ts @@ -24,7 +24,7 @@ // carrying its tarball. CL-7071 moved that publish off `seedTenant` // onto `workbench setup` (the root tenant; descendants inherit). The // connect flow runs on the genesis root itself, so an explicit -// `publishCorbitsToolsRegistry` hop onto the root stands in for +// `runPublishTools` hop onto the root stands in for // setup, then `ensureSeeded` deploys without packing. // // Stubbing note: onboarding's own `POST /api/onboarding/complete` route @@ -62,9 +62,9 @@ import { createGitWorkflowPusher, DEFAULT_WORKFLOWS, isLiveDeploymentStatus, - publishCorbitsToolsRegistry, seedTenant, } from "../../packages/seeding/src/index.ts"; +import { runPublishTools } from "../publish-tools.ts"; import { createHubAPI, parseAs, @@ -391,19 +391,24 @@ describe.skipIf(databaseUrl === undefined)( } // CL-7071: seedTenant/ensureSeeded no longer pack. The connect - // flow runs on the genesis root itself, so publish - // `corbits-tools` onto the root the way `workbench setup` does. - // Then ensureSeeded deploys assistant without packing. + // flow runs on the genesis root itself, so install `corbits-tools` + // onto that already-existing tenant with the publish-tools CLI — + // the same sign-in → tenant resolve → publish path an operator + // runs (`bun run publish-tools`) — rather than calling the + // publisher with a privileged cookie jar directly. Then + // ensureSeeded deploys assistant without packing. await hop( - "publish corbits-tools onto the provisioned root bench (setup's job, not seed's)", + "publish-tools installs corbits-tools onto the provisioned root bench (setup's job, not seed's)", async () => { - await publishCorbitsToolsRegistry({ - api: hubApi, - cookies: admin.cookies, + const result = await runPublishTools({ hubUrl: hub.baseUrl, - tenantId: tenant.tenantId, + email: "alice@example.com", + password: "password123", + tenant: provisioned.tenantSlug, log: () => undefined, }); + expect(result.success).toBe(true); + expect(result.summaries.length).toBeGreaterThan(0); }, ); diff --git a/scripts/publish-tools.test.ts b/scripts/publish-tools.test.ts new file mode 100644 index 000000000..5e178d298 --- /dev/null +++ b/scripts/publish-tools.test.ts @@ -0,0 +1,193 @@ +// Unit gates for `bun run publish-tools`: a stub hub API proves the +// command signs in (never signs up), resolves the target tenant from +// `--tenant` or the admin's sole membership, and invokes the existing +// publisher with the signed-in cookies. The publisher itself is proven +// by `packages/tool-registry-publish`'s suite and the local-rip e2e hop. +import { describe, expect, test } from "bun:test"; +import { runPublishTools, type PublishToolsArgs } from "./publish-tools.ts"; + +type ApiStub = ( + method: string, + path: string, +) => Promise<{ status: number; data: unknown; cookies: string[] }>; + +function principalsRow(tenant: { + id: string; + slug: string; + name: string; +}): unknown { + return { + principalId: `principal_${tenant.id}`, + tenantId: tenant.id, + tenantName: tenant.name, + tenantSlug: tenant.slug, + kind: "user", + status: "active", + roles: [{ id: "role_owner", name: "owner" }], + }; +} + +function stubApi(principals: unknown[]): { + api: ApiStub; + calls: { method: string; path: string }[]; +} { + const calls: { method: string; path: string }[] = []; + const api: ApiStub = (method, path) => { + calls.push({ method, path }); + if (path === "/api/auth/sign-in/email") { + return Promise.resolve({ + status: 200, + data: { user: { id: "user_admin" } }, + cookies: ["better-auth.session_token=stub"], + }); + } + if (path === "/api/me/principals") { + return Promise.resolve({ + status: 200, + data: { data: principals, nextCursor: null }, + cookies: [], + }); + } + if (path.includes("/assets?")) { + return Promise.resolve({ status: 200, data: [], cookies: [] }); + } + if (method === "POST" && path === "/api/tenants/tenant_genesis/assets") { + return Promise.resolve({ + status: 201, + data: { + id: "asset_registry", + kind: "package-registry", + name: "corbits-tools", + displayName: null, + tenantId: "tenant_genesis", + creatorPrincipalId: null, + createdAt: new Date(0).toISOString(), + updatedAt: new Date(0).toISOString(), + }, + cookies: [], + }); + } + if (path.includes("/tarballs/")) { + return Promise.resolve({ + status: 200, + data: { commit: "sha_stub", integrity: "sha512-stub" }, + cookies: [], + }); + } + return Promise.resolve({ status: 404, data: null, cookies: [] }); + }; + return { api, calls }; +} + +function args( + api: ApiStub, + tenant?: string, + calls?: { method: string; path: string }[], +): PublishToolsArgs { + return { + hubUrl: "http://localhost:3000", + email: "admin@example.com", + password: "password123", + ...(tenant !== undefined ? { tenant } : {}), + log: () => undefined, + checkFreshness: () => Promise.resolve(), + packageDirs: ["/stub/package"], + fetchImpl: (input, init) => { + calls?.push({ + method: init.method ?? "GET", + path: new URL(input).pathname, + }); + return Promise.resolve( + new Response( + JSON.stringify({ commit: "sha_stub", integrity: "sha512-stub" }), + { status: 200 }, + ), + ); + }, + pack: () => + Promise.resolve({ + name: "@corbits/memory-tools", + version: "0.0.1", + filename: "corbits-memory-tools-0.0.1.tgz", + bytes: new TextEncoder().encode("tarball-bytes"), + }), + api: api as never, + }; +} + +describe("runPublishTools", () => { + test("signs in, resolves the sole membership, and publishes onto it", async () => { + const { api, calls } = stubApi([ + principalsRow({ + id: "tenant_genesis", + slug: "genesis-root", + name: "Genesis Root", + }), + ]); + const puts: { method: string; path: string }[] = []; + const result = await runPublishTools(args(api, undefined, puts)); + expect(result.success).toBe(true); + expect( + calls.some( + (call) => + call.method === "POST" && call.path === "/api/auth/sign-in/email", + ), + ).toBe(true); + expect( + calls.some( + (call) => + call.method === "POST" && + call.path === "/api/tenants/tenant_genesis/assets", + ), + ).toBe(true); + expect( + puts.some( + (call) => + call.method === "PUT" && + call.path === + "/api/tenants/tenant_genesis/assets/asset_registry/tarballs/corbits-memory-tools-0.0.1.tgz", + ), + ).toBe(true); + }); + + test("honors --tenant by slug", async () => { + const { api, calls } = stubApi([ + principalsRow({ + id: "tenant_genesis", + slug: "genesis-root", + name: "Genesis Root", + }), + ]); + await runPublishTools(args(api, "genesis-root")); + expect(calls.some((call) => call.path.includes("tenant_genesis"))).toBe( + true, + ); + }); + + test("fails when --tenant matches no membership", async () => { + const { api } = stubApi([ + principalsRow({ + id: "tenant_genesis", + slug: "genesis-root", + name: "Genesis Root", + }), + ]); + await expect(runPublishTools(args(api, "nope"))).rejects.toThrow( + /no tenant matching "nope"/, + ); + }); + + test("fails when the admin belongs to several tenants and no --tenant is given", async () => { + const { api } = stubApi([ + principalsRow({ + id: "tenant_genesis", + slug: "genesis-root", + name: "Genesis Root", + }), + principalsRow({ id: "tenant_other", slug: "other", name: "Other" }), + ]); + await expect(runPublishTools(args(api))).rejects.toThrow( + /--tenant is required/, + ); + }); +}); diff --git a/scripts/publish-tools.ts b/scripts/publish-tools.ts new file mode 100644 index 000000000..51137a91d --- /dev/null +++ b/scripts/publish-tools.ts @@ -0,0 +1,153 @@ +// `bun run publish-tools` — packs every `@corbits/*-tools` package and +// publishes it into an existing tenant's `corbits-tools` package-registry +// asset over the hub's native REST routes. Sign-in only, never sign-up: +// the tenant must already exist (the genesis signup or `bun run dev`'s +// seeded admin created it); this command only installs the registry +// onto it. No daemon, no new route, no new table. +// +// Environment: HUB_ADMIN_EMAIL and HUB_ADMIN_PASSWORD name the +// signing-in admin; `--tenant ` picks the target tenant +// (defaults to the admin's only membership when exactly one). +import { type } from "arktype"; +import { PrincipalSummary, paginatedSchema } from "@intx/types"; +// Relative imports, the same convention scripts/e2e uses: the root +// package.json does not depend on these workspace packages, only the +// hub-facing products do. +import { + createHubAPI, + parseAs, + signIn, + type ApiCall, +} from "../packages/hub-api-client/src/index.ts"; +import { + publishCorbitsToolsRegistry, + type PackedTarball, + type PublishCorbitsToolsRegistryResult, +} from "../packages/tool-registry-publish/src/index.ts"; + +export type PublishToolsArgs = { + hubUrl: string; + email: string; + password: string; + /** Target tenant id or slug; required when the admin belongs to more than one tenant. */ + tenant?: string; + log?: (line: string) => void; + /** Test seams, mirroring `publishCorbitsToolsRegistry`'s. */ + api?: ApiCall; + fetchImpl?: (input: string, init: RequestInit) => Promise; + checkFreshness?: () => Promise; + packageDirs?: readonly string[]; + pack?: (packageDir: string) => Promise; +}; + +async function resolveTenantId( + api: ApiCall, + cookies: string[], + tenant: string | undefined, +): Promise { + const response = await api("GET", "/api/me/principals", undefined, cookies); + const summary = parseAs( + paginatedSchema(PrincipalSummary), + response.data, + "principals response", + ); + if (tenant !== undefined) { + const named = summary.data.find( + (principal) => + principal.tenantId === tenant || + principal.tenantSlug === tenant || + principal.tenantName === tenant, + ); + if (named === undefined) { + throw new Error( + `publish-tools: no tenant matching "${tenant}" in the signed-in admin's memberships (${summary.data.map((principal) => principal.tenantSlug).join(", ")})`, + ); + } + return named.tenantId; + } + if (summary.data.length !== 1) { + throw new Error( + `publish-tools: --tenant is required when the admin belongs to ${summary.data.length} tenants (${summary.data.map((principal) => principal.tenantSlug).join(", ")})`, + ); + } + const only = summary.data[0]; + if (only === undefined) { + throw new Error( + "publish-tools: the signed-in admin belongs to no tenant; sign up or seed an account first", + ); + } + return only.tenantId; +} + +export async function runPublishTools( + args: PublishToolsArgs, +): Promise { + const log = args.log ?? ((): void => undefined); + const api = args.api ?? createHubAPI(args.hubUrl); + const session = await signIn(api, { + email: args.email, + password: args.password, + }); + log(`signed in as ${args.email}`); + const tenantId = await resolveTenantId(api, session.cookies, args.tenant); + log(`publishing the corbits-tools registry onto tenant ${tenantId}`); + return publishCorbitsToolsRegistry({ + api, + cookies: session.cookies, + hubUrl: args.hubUrl, + tenantId, + log, + ...(args.checkFreshness !== undefined + ? { checkFreshness: args.checkFreshness } + : {}), + ...(args.fetchImpl !== undefined ? { fetchImpl: args.fetchImpl } : {}), + ...(args.packageDirs !== undefined + ? { packageDirs: args.packageDirs } + : {}), + ...(args.pack !== undefined ? { pack: args.pack } : {}), + }); +} + +const Args = type({ + "tenant?": "string", +}); + +async function main(): Promise { + const email = process.env["HUB_ADMIN_EMAIL"]; + const password = process.env["HUB_ADMIN_PASSWORD"]; + if ( + email === undefined || + email === "" || + password === undefined || + password === "" + ) { + console.error( + "publish-tools: set HUB_ADMIN_EMAIL and HUB_ADMIN_PASSWORD to an existing admin account, and start the stack with `bun run dev` first.", + ); + process.exit(1); + } + const raw = process.argv.slice(2); + const tenantIndex = raw.indexOf("--tenant"); + const tenant = + tenantIndex >= 0 ? (raw[tenantIndex + 1] ?? undefined) : undefined; + const parsed = Args({ tenant }); + if (parsed instanceof type.errors) { + console.error(`publish-tools: ${parsed.summary}`); + process.exit(1); + } + const hubUrl = process.env["BASE_URL"] ?? "http://localhost:3000"; + const result = await runPublishTools({ + hubUrl, + email, + password, + ...(parsed.tenant !== undefined ? { tenant: parsed.tenant } : {}), + log: (line) => console.log(`[publish-tools] ${line}`), + }); + console.log( + `[publish-tools] done: ${result.summaries.length} tarball(s) uploaded`, + ); +} + +if (import.meta.main) { + main(); +}