From 09ebbfa0d054fc087e961ffd9073e5b59bd438a2 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sat, 3 Oct 2026 12:38:53 +0000 Subject: [PATCH 1/2] ruvnet-brain: content-addressed corpus releases, sig/receipt recorded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream stuinfla/ruvnet-brain now publishes signed, content-addressed generations (tags corpus-sha256-, with .zip.sig and corpus-receipt.json). The ingest now: - selects GitHub's designated /releases/latest, falling back to the list only when that release lacks the zip. The list API orders by creation time and put runtime release v4.5.2 ahead of the promoted corpus-sha256-832bae01…, so first-in-list picked the wrong corpus; - for corpus-sha256-* tags, requires the published .zip.sha256 to equal the tag digest before downloading, and the bytes to hash to it; - fetches .zip.sig and corpus-receipt.json beside the zip, refuses a receipt whose archive digest differs from the zip, or whose own digest differs from the release body's "Receipt SHA-256"; - records sig_present/sig_bytes, receipt_sha256, receipt_matches_archive, archive_sha256, content_addressed and shipped_runtime in the manifest, and keeps the full tag as corpus_version. signature_verified is always false ("no published public key"): no key is guessed; - is import-safe (entry guard, pg loaded in main) so the pure helpers are unit-tested in tests/ruvnet-brain-ingest.spec.mjs (16 tests, node --test). ruvnet_brain_status gains a derived provenance block. The skill documents the tag scheme, the provenance fields, that `npx ruvnet-brain` (RVF into ~/.cache) is not our path, the isovision.ai docs link, and the serial REINDEX the ingest does not perform. Co-Authored-By: jjohare --- mcp/ruvnet-brain/server.js | 17 +- scripts/ruvnet-brain-ingest.mjs | 276 ++++++++++++++++++++++------- skills/ruvnet-brain/SKILL.md | 56 ++++-- tests/ruvnet-brain-ingest.spec.mjs | 186 +++++++++++++++++++ 4 files changed, 455 insertions(+), 80 deletions(-) create mode 100644 tests/ruvnet-brain-ingest.spec.mjs diff --git a/mcp/ruvnet-brain/server.js b/mcp/ruvnet-brain/server.js index 096d82df6..bd96597bc 100644 --- a/mcp/ruvnet-brain/server.js +++ b/mcp/ruvnet-brain/server.js @@ -188,6 +188,21 @@ async function brainStatus() { [NAMESPACE], ); out.manifest = manifest.rows.length ? parseVal(manifest.rows[0].value) : null; + // Release provenance, as recorded by the ingest (scripts/ruvnet-brain-ingest.mjs). + // A manifest from a pre-2026-10-03 ingest carries none of these fields. + const m = out.manifest && typeof out.manifest === 'object' ? out.manifest : null; + if (m && 'sig_present' in m) { + out.provenance = { + content_addressed: Boolean(m.content_addressed), + archive_sha256: m.archive_sha256 ?? null, + sig_present: Boolean(m.sig_present), + signature_verified: Boolean(m.signature_verified), + signature_reason: m.signature_reason ?? null, + receipt_sha256: m.receipt_sha256 ?? null, + receipt_matches_archive: m.receipt_matches_archive ?? null, + shipped_runtime: m.shipped_runtime ?? null, + }; + } if (!out.corpus.total) { out.hint = 'Corpus empty — run scripts/ruvnet-brain-ingest.mjs (auto-runs at boot when [skills.ruvnet_brain].auto_ingest = true).'; } @@ -219,7 +234,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({ }, { name: 'ruvnet_brain_status', - description: 'Corpus health: chunk counts, embedded coverage, distinct repos, ingest manifest (corpus version + timestamp).', + description: 'Corpus health: chunk counts, embedded coverage, distinct repos, ingest manifest (corpus version + timestamp) and release provenance (archive digest, signature present/verified, receipt digest, shipped runtime).', inputSchema: { type: 'object', properties: {} }, }, ], diff --git a/scripts/ruvnet-brain-ingest.mjs b/scripts/ruvnet-brain-ingest.mjs index bf4c59f19..6c173ff16 100644 --- a/scripts/ruvnet-brain-ingest.mjs +++ b/scripts/ruvnet-brain-ingest.mjs @@ -27,11 +27,24 @@ // failures exceed RUVNET_BRAIN_MAX_FAILED_CHUNKS (default 1000). Safe to // re-run at every boot / after every build — this IS the "bleeding edge at // build time" reconciliation. +// +// Content-addressed generations (upstream since 2026-10-03): release tags are +// `corpus-sha256-`. For those the published .zip.sha256 MUST +// equal the tag digest (checked before the download) and the downloaded bytes +// must hash to it. The detached .zip.sig and corpus-receipt.json are fetched +// beside the zip; the receipt's archive digest must equal the zip digest and, +// when the release body states a "Receipt SHA-256", the receipt must hash to +// it. The signature is RECORDED (sig_present) but never claimed verified: +// upstream publishes no public key, and we do not guess one +// (signature_verified=false, signature_reason="no published public key"). +// Upstream's own delivery (`npx ruvnet-brain` → RVF files in ~/.cache) is +// not this path; agentbox ingests the zip's passages into ruvector-postgres. import { createHash } from 'node:crypto'; import { createRequire } from 'node:module'; -import { existsSync, mkdirSync, readdirSync, createReadStream, rmSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { mkdirSync, readdirSync, readFileSync, realpathSync, createReadStream, rmSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { createInterface } from 'node:readline'; import { execFileSync } from 'node:child_process'; import http from 'node:http'; @@ -73,7 +86,6 @@ function loadPg() { for (const c of candidates) { try { return req(c); } catch { /* next */ } } die('pg module not found in any baked closure'); } -const { Pool } = loadPg(); // ── Xinference client (batch) ──────────────────────────────────────────────── function httpJson(url, { method = 'GET', body = null, timeout = 60000, headers = {} } = {}) { @@ -147,27 +159,124 @@ const wellFormed = (s) => { // again after the cut. const embedText = (s) => wellFormed(s.substring(0, EMBED_TRUNC)); -// ── Release digest verification ────────────────────────────────────────────── -// Upstream publishes .sha256 alongside the zip. A mismatch is fatal -// (this corpus is treated as ground truth by the grounding hook); a release -// without a digest asset only warns unless RUVNET_BRAIN_REQUIRE_DIGEST=1. -function verifyDigest(zipPath, releaseUrl) { - let expected = null; - try { - const out = execFileSync('curl', ['-fsSL', '--retry', '2', '--max-time', '60', `${releaseUrl}.sha256`], { encoding: 'utf8' }); - expected = (out.match(/\b[0-9a-f]{64}\b/i) || [null])[0]; - } catch { /* digest asset absent or unreachable */ } - if (!expected) { - if (process.env.RUVNET_BRAIN_REQUIRE_DIGEST === '1') die(`no .sha256 digest at ${releaseUrl}.sha256 and RUVNET_BRAIN_REQUIRE_DIGEST=1`); - log('WARN: no .sha256 digest published for this release — skipping download verification'); - return; +// ── Release provenance (pure helpers, unit-tested) ────────────────────────── +const HEX64 = /\b[0-9a-f]{64}\b/i; +export const SIGNATURE_UNVERIFIED_REASON = 'no published public key'; + +/** Digest carried by a content-addressed tag `corpus-sha256-<64 hex>`, else null. */ +export function tagDigest(tag) { + const m = typeof tag === 'string' ? tag.match(/^corpus-sha256-([0-9a-f]{64})$/i) : null; + return m ? m[1].toLowerCase() : null; +} + +/** First 64-hex digest in a sha256sum-style asset body, lower-cased, else null. */ +export function parseDigest(text) { + const m = typeof text === 'string' ? text.match(HEX64) : null; + return m ? m[0].toLowerCase() : null; +} + +/** + * Content-addressing proof. A `corpus-sha256-*` tag IS the archive digest, so + * the release's published .sha256 must exist and equal it; anything else is + * refused. Legacy tags (v4.x) are not content-addressed and pass through. + */ +export function checkContentAddress(tag, publishedDigest) { + const want = tagDigest(tag); + if (!want) return { contentAddressed: false, ok: true, reason: null }; + const got = publishedDigest ? String(publishedDigest).toLowerCase() : null; + if (!got) return { contentAddressed: true, ok: false, reason: `content-addressed tag ${tag} but no published .sha256 digest` }; + if (got !== want) return { contentAddressed: true, ok: false, reason: `published .sha256 ${got} does not equal tag digest ${want}` }; + return { contentAddressed: true, ok: true, reason: null }; +} + +/** URL of another asset in the same release directory as `assetUrl`. */ +export function siblingAssetUrl(assetUrl, name) { + const u = new URL(assetUrl); + u.pathname = u.pathname.replace(/[^/]*$/, encodeURIComponent(name)); + u.search = ''; u.hash = ''; + return u.toString(); +} + +/** Upstream runtime the corpus shipped with: receipt first, then release body. */ +export function shippedRuntime(receipt, releaseBody) { + if (receipt && typeof receipt === 'object') { + if (typeof receipt.archiveManifestReleaseTag === 'string' && receipt.archiveManifestReleaseTag) return receipt.archiveManifestReleaseTag; + if (typeof receipt.archiveManifestVersion === 'string' && receipt.archiveManifestVersion) { + return receipt.archiveManifestVersion.startsWith('v') ? receipt.archiveManifestVersion : `v${receipt.archiveManifestVersion}`; + } } - const actual = execFileSync('sha256sum', [zipPath], { encoding: 'utf8' }).split(/\s+/)[0]; - if (actual.toLowerCase() !== expected.toLowerCase()) { - rmSync(zipPath, { force: true }); - die(`download sha256 mismatch: expected ${expected}, got ${actual} — refusing to ingest`); + const m = typeof releaseBody === 'string' ? releaseBody.match(/^\s*Shipped runtime:\s*(\S+)/mi) : null; + return m ? m[1] : null; +} + +/** + * Evaluate the detached signature and corpus receipt against the downloaded + * archive digest. Returns the manifest provenance fields plus ok/reason; a + * receipt that names a different archive, does not parse, or does not hash to + * the release body's "Receipt SHA-256" is refused. The signature is recorded + * only — there is no published key to verify it against. + */ +export function evaluateProvenance({ zipSha256, receiptText, sigBytes, releaseBody }) { + const out = { + ok: true, reason: null, + sig_present: Boolean(sigBytes && sigBytes.length), + sig_bytes: sigBytes ? sigBytes.length : 0, + signature_verified: false, + signature_reason: SIGNATURE_UNVERIFIED_REASON, + receipt_sha256: null, receipt_archive_sha256: null, receipt_matches_archive: null, + shipped_runtime: null, + }; + const refuse = (reason) => { out.ok = false; out.reason = reason; return out; }; + let receipt = null; + if (receiptText != null) { + out.receipt_sha256 = createHash('sha256').update(receiptText).digest('hex'); + try { receipt = JSON.parse(receiptText); } catch (e) { return refuse(`corpus receipt does not parse (${e.message})`); } + const archived = parseDigest(receipt?.archive?.sha256 ?? receipt?.archiveSha256 ?? ''); + out.receipt_archive_sha256 = archived; + out.receipt_matches_archive = Boolean(archived && zipSha256 && archived === String(zipSha256).toLowerCase()); + if (!out.receipt_matches_archive) { + return refuse(`receipt archive sha256 ${archived || 'missing'} does not equal downloaded zip sha256 ${zipSha256}`); + } + const stated = typeof releaseBody === 'string' ? releaseBody.match(/Receipt SHA-256:\s*([0-9a-f]{64})/i) : null; + if (stated && stated[1].toLowerCase() !== out.receipt_sha256) { + return refuse(`receipt sha256 ${out.receipt_sha256} does not equal the release body's Receipt SHA-256 ${stated[1].toLowerCase()}`); + } } - log(`download digest verified (sha256 ${expected.slice(0, 12)}…)`); + out.shipped_runtime = shippedRuntime(receipt, releaseBody); + return out; +} + +/** Manifest row value. corpus_version is always the full release tag. */ +export function buildManifest({ version, stats, source, archiveSha256, provenance, urn = null, now = new Date().toISOString() }) { + const p = provenance || {}; + return { + corpus_version: version, + ingested_at: now, + chunks: stats.chunks, embedded: stats.embedded, unchanged: stats.unchanged, + pruned: stats.pruned, failed_chunks: stats.failed_chunks, + source, + content_addressed: Boolean(tagDigest(version)), + archive_sha256: archiveSha256 || null, + sig_present: Boolean(p.sig_present), + sig_bytes: p.sig_bytes || 0, + signature_verified: false, + signature_reason: p.signature_reason || SIGNATURE_UNVERIFIED_REASON, + receipt_sha256: p.receipt_sha256 ?? null, + receipt_matches_archive: p.receipt_matches_archive ?? null, + shipped_runtime: p.shipped_runtime ?? null, + ...(urn ? { dataset_urn: urn } : {}), + }; +} + +// ── Release asset fetches (curl; absent asset → null) ──────────────────────── +function fetchText(url) { + try { return execFileSync('curl', ['-fsSL', '--retry', '2', '--max-time', '60', url], { encoding: 'utf8' }); } catch { return null; } +} +function fetchBytes(url, dest) { + try { + execFileSync('curl', ['-fsSL', '--retry', '2', '--max-time', '120', '-o', dest, url], { stdio: ['ignore', 'ignore', 'ignore'] }); + return readFileSync(dest); + } catch { rmSync(dest, { force: true }); return null; } } // ── Release version discovery ──────────────────────────────────────────────── @@ -192,51 +301,60 @@ function discoverVersion(url) { }); } -// GitHub can mark a release "latest" before its large corpus asset finishes -// uploading (v4.3.1 was observed with zero assets). The convenience -// /latest/download URL then returns 404 even though the previous complete -// release remains usable. Resolve that URL through the releases API and pick -// the newest release that actually contains the requested asset. This keeps -// version reconciliation tied to downloadable bytes instead of a bare tag. +// Release selection. GitHub's designated latest (/releases/latest) is +// authoritative: upstream stages a content-addressed generation as a +// prerelease and promotes it to latest only after a clean customer install. +// The list API orders by creation time, NOT by that designation — on +// 2026-10-03 it put runtime release v4.5.2 ahead of the promoted +// corpus-sha256-832bae01… (same created_at), so "first in list" picked the +// wrong corpus. The list is only the fallback for when the designated release +// lacks the asset (v4.3.1 was marked latest before its zip finished uploading). +const assetOf = (release, assetName) => (release && !release.draft && release.tag_name && Array.isArray(release.assets) + ? release.assets.find((a) => a?.name === assetName && a?.browser_download_url) || null + : null); + +export function pickRelease(latest, releases, assetName) { + const hit = assetOf(latest, assetName); + if (hit) return { url: hit.browser_download_url, version: latest.tag_name, body: typeof latest.body === 'string' ? latest.body : '', fallback: false }; + for (const release of Array.isArray(releases) ? releases : []) { + const asset = assetOf(release, assetName); + if (asset) return { url: asset.browser_download_url, version: release.tag_name, body: typeof release.body === 'string' ? release.body : '', fallback: true }; + } + return { url: null, version: null, body: '', fallback: true }; +} + async function resolveRelease(url) { const pinned = url.match(/\/releases\/download\/([^/]+)\//); - if (pinned) return { url, version: decodeURIComponent(pinned[1]) }; + if (pinned) return { url, version: decodeURIComponent(pinned[1]), body: '' }; - const latest = url.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/latest\/download\/([^/?#]+)(?:[?#].*)?$/); - if (latest) { - const [, owner, repo, encodedAsset] = latest; + const latestUrl = url.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/latest\/download\/([^/?#]+)(?:[?#].*)?$/); + if (latestUrl) { + const [, owner, repo, encodedAsset] = latestUrl; const assetName = decodeURIComponent(encodedAsset); + const api = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/releases`; + const headers = { + Accept: 'application/vnd.github+json', + 'User-Agent': 'agentbox-ruvnet-brain-ingest', + 'X-GitHub-Api-Version': '2022-11-28', + }; try { - const releases = await httpJson( - `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/releases?per_page=20`, - { - timeout: 30000, - headers: { - Accept: 'application/vnd.github+json', - 'User-Agent': 'agentbox-ruvnet-brain-ingest', - 'X-GitHub-Api-Version': '2022-11-28', - }, - }, - ); - for (const release of Array.isArray(releases) ? releases : []) { - if (release?.draft) continue; - const asset = Array.isArray(release?.assets) - ? release.assets.find((candidate) => candidate?.name === assetName) - : null; - if (!asset?.browser_download_url || !release?.tag_name) continue; - if (release.tag_name !== releases[0]?.tag_name) { - log(`WARN: latest GitHub release ${releases[0]?.tag_name || 'unknown'} has no ${assetName}; using newest complete release ${release.tag_name}`); - } - return { url: asset.browser_download_url, version: release.tag_name }; + let latest = null; + try { latest = await httpJson(`${api}/latest`, { timeout: 30000, headers }); } catch (error) { + log(`WARN: GitHub /releases/latest lookup failed (${error.message}); scanning the release list`); } - log(`WARN: no downloadable ${assetName} found in the 20 newest GitHub releases`); - return { url: null, version: null }; + const releases = assetOf(latest, assetName) ? [] : await httpJson(`${api}?per_page=20`, { timeout: 30000, headers }); + const picked = pickRelease(latest, releases, assetName); + if (picked.version && picked.fallback) { + log(`WARN: designated latest release ${latest?.tag_name || 'unknown'} has no ${assetName}; using newest complete release ${picked.version}`); + } + if (!picked.version) log(`WARN: no downloadable ${assetName} found in the designated latest or the 20 newest GitHub releases`); + return picked; } catch (error) { log(`WARN: GitHub release API lookup failed (${error.message}); falling back to the configured URL`); } } - return { url, version: await discoverVersion(url) }; + return { url, version: await discoverVersion(url), body: '' }; } // ── SQL helpers (mirror ruvector-mcp.cjs conventions) ──────────────────────── @@ -294,6 +412,7 @@ function poolConfig(conninfo) { async function main() { if (!CONNINFO) die('RUVECTOR_PG_CONNINFO not set'); + const { Pool } = loadPg(); const pool = new Pool({ ...poolConfig(CONNINFO), max: 4 }); const manifestRow = async () => { @@ -345,9 +464,34 @@ async function main() { const extractDir = join(STAGING, 'passages'); rmSync(extractDir, { recursive: true, force: true }); mkdirSync(extractDir, { recursive: true }); + // 3a. Published digest first: for a content-addressed tag it must equal the + // tag digest, so a mislabelled release is refused before 600+ MB moves. + const publishedDigest = parseDigest(fetchText(`${release.url}.sha256`)); + const ca = checkContentAddress(version, publishedDigest); + if (!ca.ok) die(`${ca.reason} — refusing to ingest`); + if (ca.contentAddressed) log(`content-addressed release: published .sha256 equals tag digest (${publishedDigest.slice(0, 12)}…)`); + if (!publishedDigest) { + if (process.env.RUVNET_BRAIN_REQUIRE_DIGEST === '1') die(`no .sha256 digest at ${release.url}.sha256 and RUVNET_BRAIN_REQUIRE_DIGEST=1`); + log('WARN: no .sha256 digest published for this release — skipping download verification'); + } log(`downloading ${release.url} → ${zipPath} (hundreds of MB, be patient)`); execFileSync('curl', ['-fSL', '--retry', '3', '--max-time', '1800', '-o', zipPath, release.url], { stdio: ['ignore', 'ignore', 'inherit'] }); - verifyDigest(zipPath, release.url); + const zipSha256 = execFileSync('sha256sum', [zipPath], { encoding: 'utf8' }).split(/\s+/)[0].toLowerCase(); + if (publishedDigest && zipSha256 !== publishedDigest) { + rmSync(zipPath, { force: true }); + die(`download sha256 mismatch: expected ${publishedDigest}, got ${zipSha256} — refusing to ingest`); + } + if (publishedDigest) log(`download digest verified (sha256 ${zipSha256.slice(0, 12)}…)`); + + // 3b. Detached signature + corpus receipt, kept beside the zip in staging. + const sigBytes = fetchBytes(`${release.url}.sig`, join(STAGING, 'ruvnet-brain.zip.sig')); + const receiptBytes = fetchBytes(siblingAssetUrl(release.url, 'corpus-receipt.json'), join(STAGING, 'corpus-receipt.json')); + const provenance = evaluateProvenance({ + zipSha256, receiptText: receiptBytes ? receiptBytes.toString('utf8') : null, sigBytes, releaseBody: release.body, + }); + if (!provenance.ok) { rmSync(zipPath, { force: true }); die(`${provenance.reason} — refusing to ingest`); } + log(`provenance: sig ${provenance.sig_present ? `present (${provenance.sig_bytes} B, NOT verified: ${provenance.signature_reason})` : 'absent'}; ` + + `receipt ${provenance.receipt_sha256 ? `${provenance.receipt_sha256.slice(0, 12)}… matches archive` : 'absent'}; shipped runtime ${provenance.shipped_runtime || 'unknown'}`); log('extracting *.passages.jsonl'); try { // Exclude macOS AppleDouble junk (__MACOSX/, ._*) — upstream zips are built @@ -479,13 +623,11 @@ async function main() { // 7. Manifest stamp (+ best-effort ADR-013 dataset URN). const urn = mintCorpusUrn(version); - const manifestValue = { - corpus_version: version, - ingested_at: new Date().toISOString(), - chunks: seenKeys.size, embedded: inserted, unchanged: skipped, - pruned: pruned.rowCount, failed_chunks: failedChunks, - source: RELEASE_URL, ...(urn ? { dataset_urn: urn } : {}), - }; + const manifestValue = buildManifest({ + version, + stats: { chunks: seenKeys.size, embedded: inserted, unchanged: skipped, pruned: pruned.rowCount, failed_chunks: failedChunks }, + source: RELEASE_URL, archiveSha256: zipSha256, provenance, urn, + }); await pool.query( `INSERT INTO memory_entries (id, namespace, key, value, source_type, metadata, embedding) VALUES ($1, $2, 'ruvnet/manifest', $3::jsonb, $4, $5::jsonb, NULL) @@ -500,4 +642,8 @@ async function main() { await pool.end(); } -main().catch((e) => die(e.stack || e.message)); +// Entry guard: importing this module (unit tests) must not run an ingest. +const realOrSelf = (p) => { try { return realpathSync(p); } catch { return p; } }; +const invokedDirectly = Boolean(process.argv[1]) + && realOrSelf(resolve(process.argv[1])) === realOrSelf(fileURLToPath(import.meta.url)); +if (invokedDirectly) main().catch((e) => die(e.stack || e.message)); diff --git a/skills/ruvnet-brain/SKILL.md b/skills/ruvnet-brain/SKILL.md index 008a40bf7..d4ab4f34b 100644 --- a/skills/ruvnet-brain/SKILL.md +++ b/skills/ruvnet-brain/SKILL.md @@ -7,7 +7,7 @@ description: > alternative (Pinecone, LangChain, ChromaDB, hnswlib): query the search_ruvnet MCP tool to ground the answer in indexed source code rather than training-data guesses. -version: 0.3.0 +version: 0.4.0 related_skills: - lazy-fetch - agentdb-memory-patterns @@ -48,26 +48,54 @@ so this guard is firm. ## Keeping the corpus current (periodic update playbook) The upstream corpus is a GitHub release asset that moves independently of this -deployment (`stuinfla/ruvnet-brain` `releases/latest/download/ruvnet-brain.zip`). -The ingest is fully idempotent: it discovers the latest tag via the release -redirect, fast no-ops when the stamped `corpus_version` matches, and on a real -bump embeds only new/changed chunks (content-addressed keys) and prunes rows -absent from the new corpus. The 2026-08-14 v3.3.1→v4.0.36 reconcile embedded -6,259 and pruned 1,807 of 136,439 chunks in ~7 min. +deployment (`stuinfla/ruvnet-brain` `releases/latest/download/ruvnet-brain.zip`; +upstream docs: ). The ingest is fully +idempotent: it resolves the newest release that actually carries the zip, +fast no-ops when the stamped `corpus_version` matches, and on a real bump +embeds only new/changed chunks (content-addressed keys) and prunes rows absent +from the new corpus. The 2026-08-14 v3.3.1→v4.0.36 reconcile embedded 6,259 +and pruned 1,807 of 136,439 chunks in ~7 min. + +**Content-addressed generations (since 2026-10-03).** Upstream tags are now +`corpus-sha256-` (e.g. `corpus-sha256-832bae01…`, shipped +runtime v4.5.2) instead of `v4.x`. Each release carries `ruvnet-brain.zip`, +`.zip.sha256`, `.zip.sig` (64-byte detached signature) and +`corpus-receipt.json`. The ingest treats the tag as an opaque version (the +manifest keeps the **full** tag) and enforces: + +- the published `.zip.sha256` must **equal the tag digest** (checked before + the download; refused otherwise), and the downloaded bytes must hash to it; +- the receipt's `archive.sha256` must equal the zip digest, and the receipt + must hash to the release body's `Receipt SHA-256` when stated (refused + otherwise); +- the `.sig` is **recorded, not verified**: upstream publishes no public key, + so `signature_verified` is always `false` with + `signature_reason: "no published public key"`. Never guess or hand-roll a + key; wire verification only once upstream publishes one. + +`ruvnet_brain_status` (and `./agentbox.sh ruvnet-brain status`) show these in +the manifest: `corpus_version`, `content_addressed`, `archive_sha256`, +`sig_present`, `sig_bytes`, `signature_verified`, `signature_reason`, +`receipt_sha256`, `receipt_matches_archive`, `shipped_runtime`. The `.sig` and +receipt stay in `RUVNET_BRAIN_STAGING` beside the (deleted) zip for inspection. + +**`npx ruvnet-brain` is not our path.** Upstream's own delivery installs RVF +stores into `~/.cache` and runs its own embedder. Agentbox deliberately takes +only the zip's `*.passages.jsonl` into ruvector-postgres (`ruvnet-kb`, our +bge-small 384-dim space). Do not switch this deployment to npx. Procedure (run whenever staleness is suspected; safe to run any time): ```bash ./agentbox.sh ruvnet-brain status # compare manifest corpus_version vs upstream ./agentbox.sh ruvnet-brain ingest # reconcile (no-op if current) -# AFTER any real delta (embedded+pruned > 0): the index-law applies — -# non-concurrent HNSW rebuild, NEVER CREATE INDEX CONCURRENTLY on this AM: +# AFTER any real delta (embedded+pruned > 0) the index-law applies: the +# ingest does NOT rebuild the index. Non-concurrent AND serial rebuild +# (~8 min, memory writes block meanwhile); NEVER CREATE INDEX CONCURRENTLY: docker exec ruvector-postgres psql -U ruvector -d ruvector \ - -c "DROP INDEX idx_memory_embedding_hnsw;" \ - -c "CREATE INDEX idx_memory_embedding_hnsw ON public.memory_entries - USING hnsw (embedding ruvector_cosine_ops) - WITH (m='16', ef_construction='128');" # ~5-7 min; memory WRITES BLOCK meanwhile -./agentbox.sh ruvector recall # gate: frozen band must hold + -c "SET max_parallel_maintenance_workers = 0;" \ + -c "REINDEX INDEX idx_memory_embedding_hnsw;" # keeps m=16, ef_construction=128 +./agentbox.sh ruvector recall # gate: self >=175/200, true >=102/120 ``` Post-bump gate expectations (observed 2026-08-14): a corpus bump that PRUNES diff --git a/tests/ruvnet-brain-ingest.spec.mjs b/tests/ruvnet-brain-ingest.spec.mjs new file mode 100644 index 000000000..b2cc32b27 --- /dev/null +++ b/tests/ruvnet-brain-ingest.spec.mjs @@ -0,0 +1,186 @@ +// node --test unit tests for scripts/ruvnet-brain-ingest.mjs — the release +// provenance helpers for upstream's content-addressed corpus generations +// (tags `corpus-sha256-`, with .zip.sig + corpus-receipt.json). +// Pure functions only: NO DB, NO network. Importing the script is side-effect +// free (main() runs only when invoked directly; pg loads inside main()). +// +// node --test tests/ruvnet-brain-ingest.spec.mjs + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; + +import { + tagDigest, parseDigest, checkContentAddress, siblingAssetUrl, + shippedRuntime, evaluateProvenance, buildManifest, pickRelease, +} from '../scripts/ruvnet-brain-ingest.mjs'; + +const D = '832bae01c40f0dc695d4ce12eefdf1a297c2bb663437fac8566f6497d7457ec7'; +const D2 = '473a83aa6c443b6f7d18d030e1223dc89ce1074d53e8db1d9247e0e7736c777d'; +const TAG = `corpus-sha256-${D}`; +const sha = (s) => createHash('sha256').update(s).digest('hex'); +const receipt = (archiveDigest, extra = {}) => JSON.stringify({ + schemaVersion: 3, kind: 'ruvnet-brain-corpus-candidate', + archive: { file: 'ruvnet-brain.zip', sha256: archiveDigest, bytes: 679040573 }, + archiveManifestVersion: '4.5.2', archiveManifestReleaseTag: 'v4.5.2', ...extra, +}); + +// ── (a) content-addressing proof ───────────────────────────────────────────── +test('tagDigest: extracts the digest from a content-addressed tag only', () => { + assert.equal(tagDigest(TAG), D); + assert.equal(tagDigest(TAG.toUpperCase().replace('CORPUS-SHA256-', 'corpus-sha256-')), D); + assert.equal(tagDigest('v4.5.1'), null); + assert.equal(tagDigest(`corpus-sha256-${D.slice(1)}`), null); + assert.equal(tagDigest(null), null); +}); + +test('parseDigest: reads the hex digest from a sha256sum-style asset', () => { + assert.equal(parseDigest(`${D} ruvnet-brain.zip\n`), D); + assert.equal(parseDigest('not a digest'), null); + assert.equal(parseDigest(''), null); +}); + +test('checkContentAddress: content-addressed tag requires .sha256 == tag digest', () => { + assert.deepEqual(checkContentAddress(TAG, D), { contentAddressed: true, ok: true, reason: null }); + const mismatch = checkContentAddress(TAG, D2); + assert.equal(mismatch.contentAddressed, true); + assert.equal(mismatch.ok, false); + assert.match(mismatch.reason, /does not equal/); + const absent = checkContentAddress(TAG, null); + assert.equal(absent.ok, false); + assert.match(absent.reason, /no published \.sha256/); +}); + +test('checkContentAddress: legacy v-tags are not content-addressed and pass through', () => { + assert.deepEqual(checkContentAddress('v4.5.1', D), { contentAddressed: false, ok: true, reason: null }); + assert.deepEqual(checkContentAddress('v4.5.1', null), { contentAddressed: false, ok: true, reason: null }); +}); + +test('siblingAssetUrl: resolves receipt/sig beside the zip', () => { + const url = `https://github.com/stuinfla/ruvnet-brain/releases/download/${TAG}/ruvnet-brain.zip`; + assert.equal(siblingAssetUrl(url, 'corpus-receipt.json'), + `https://github.com/stuinfla/ruvnet-brain/releases/download/${TAG}/corpus-receipt.json`); + assert.equal(siblingAssetUrl(url, 'ruvnet-brain.zip.sig'), `${url}.sig`); +}); + +// ── (b) signature + receipt recorded, never "verified" without a key ──────── +test('evaluateProvenance: sig present, receipt matches archive, signature unverified', () => { + const r = receipt(D); + const p = evaluateProvenance({ zipSha256: D, receiptText: r, sigBytes: Buffer.alloc(64, 7), releaseBody: '' }); + assert.equal(p.ok, true); + assert.equal(p.sig_present, true); + assert.equal(p.sig_bytes, 64); + assert.equal(p.signature_verified, false); + assert.equal(p.signature_reason, 'no published public key'); + assert.equal(p.receipt_sha256, sha(r)); + assert.equal(p.receipt_archive_sha256, D); + assert.equal(p.receipt_matches_archive, true); +}); + +test('evaluateProvenance: receipt archive digest != zip digest is refused', () => { + const p = evaluateProvenance({ zipSha256: D, receiptText: receipt(D2), sigBytes: Buffer.alloc(64), releaseBody: '' }); + assert.equal(p.ok, false); + assert.equal(p.receipt_matches_archive, false); + assert.match(p.reason, /receipt archive sha256/); +}); + +test('evaluateProvenance: accepts the flat archiveSha256 receipt shape too', () => { + const r = JSON.stringify({ archiveSha256: D }); + const p = evaluateProvenance({ zipSha256: D, receiptText: r, sigBytes: null, releaseBody: '' }); + assert.equal(p.ok, true); + assert.equal(p.receipt_archive_sha256, D); + assert.equal(p.sig_present, false); + assert.equal(p.sig_bytes, 0); + assert.equal(p.signature_verified, false); +}); + +test('evaluateProvenance: release body Receipt SHA-256 must match the downloaded receipt', () => { + const r = receipt(D); + const good = evaluateProvenance({ zipSha256: D, receiptText: r, sigBytes: null, releaseBody: `Receipt SHA-256: ${sha(r)}\n` }); + assert.equal(good.ok, true); + const bad = evaluateProvenance({ zipSha256: D, receiptText: r, sigBytes: null, releaseBody: `Receipt SHA-256: ${D2}\n` }); + assert.equal(bad.ok, false); + assert.match(bad.reason, /release body/); +}); + +test('evaluateProvenance: unparseable receipt is refused; absent receipt is recorded as null', () => { + const bad = evaluateProvenance({ zipSha256: D, receiptText: '{not json', sigBytes: null, releaseBody: '' }); + assert.equal(bad.ok, false); + assert.match(bad.reason, /receipt/); + const none = evaluateProvenance({ zipSha256: D, receiptText: null, sigBytes: null, releaseBody: '' }); + assert.equal(none.ok, true); + assert.equal(none.receipt_sha256, null); + assert.equal(none.receipt_matches_archive, null); +}); + +// ── (c) manifest: full tag kept, shipped_runtime recorded ──────────────────── +test('shippedRuntime: receipt first, then release body, else null', () => { + assert.equal(shippedRuntime(JSON.parse(receipt(D)), ''), 'v4.5.2'); + assert.equal(shippedRuntime({ archiveManifestVersion: '4.5.2' }, ''), 'v4.5.2'); + assert.equal(shippedRuntime(null, 'Stores: 199\nShipped runtime: v4.5.2\n'), 'v4.5.2'); + assert.equal(shippedRuntime(null, ''), null); +}); + +test('buildManifest: keeps the full corpus tag and records provenance fields', () => { + const r = receipt(D); + const prov = evaluateProvenance({ zipSha256: D, receiptText: r, sigBytes: Buffer.alloc(64), releaseBody: '' }); + const m = buildManifest({ + version: TAG, stats: { chunks: 10, embedded: 3, unchanged: 7, pruned: 2, failed_chunks: 0 }, + source: 'https://example/ruvnet-brain.zip', archiveSha256: D, provenance: prov, now: '2026-10-03T13:00:00.000Z', + }); + assert.equal(m.corpus_version, TAG); + assert.equal(m.content_addressed, true); + assert.equal(m.archive_sha256, D); + assert.equal(m.sig_present, true); + assert.equal(m.signature_verified, false); + assert.equal(m.signature_reason, 'no published public key'); + assert.equal(m.receipt_sha256, sha(r)); + assert.equal(m.shipped_runtime, 'v4.5.2'); + assert.equal(m.ingested_at, '2026-10-03T13:00:00.000Z'); + assert.equal(m.embedded, 3); + assert.equal(m.pruned, 2); +}); + +test('buildManifest: legacy tag without provenance still produces a valid manifest', () => { + const m = buildManifest({ + version: 'v4.5.1', stats: { chunks: 1, embedded: 1, unchanged: 0, pruned: 0, failed_chunks: 0 }, + source: 's', archiveSha256: null, provenance: null, now: '2026-10-03T13:00:00.000Z', + }); + assert.equal(m.corpus_version, 'v4.5.1'); + assert.equal(m.content_addressed, false); + assert.equal(m.sig_present, false); + assert.equal(m.signature_verified, false); + assert.equal(m.receipt_sha256, null); + assert.equal(m.shipped_runtime, null); +}); + +// ── release selection: GitHub's designated latest wins over list order ────── +const rel = (tag, assets, extra = {}) => ({ + tag_name: tag, draft: false, body: `body of ${tag}`, + assets: assets.map((name) => ({ name, browser_download_url: `https://dl/${tag}/${name}` })), ...extra, +}); + +test('pickRelease: designated latest with the asset wins even when the list puts another release first', () => { + // Observed 2026-10-03: list API ordered v4.5.2 (same created_at) before the + // corpus-sha256 release that /releases/latest designates. + const latest = rel(TAG, ['ruvnet-brain.zip', 'ruvnet-brain.zip.sig']); + const list = [rel('v4.5.2', ['ruvnet-brain.zip']), rel('v4.5.1', ['ruvnet-brain.zip']), latest]; + const r = pickRelease(latest, list, 'ruvnet-brain.zip'); + assert.equal(r.version, TAG); + assert.equal(r.url, `https://dl/${TAG}/ruvnet-brain.zip`); + assert.equal(r.body, `body of ${TAG}`); + assert.equal(r.fallback, false); +}); + +test('pickRelease: designated latest without the asset falls back to newest complete release', () => { + const latest = rel('v4.3.1', []); // asset upload not finished + const list = [latest, rel('v4.3.0', ['ruvnet-brain.zip'])]; + const r = pickRelease(latest, list, 'ruvnet-brain.zip'); + assert.equal(r.version, 'v4.3.0'); + assert.equal(r.fallback, true); +}); + +test('pickRelease: drafts skipped; nothing usable yields nulls', () => { + const r = pickRelease(null, [rel('v9', ['ruvnet-brain.zip'], { draft: true }), rel('v8', ['other.zip'])], 'ruvnet-brain.zip'); + assert.deepEqual(r, { url: null, version: null, body: '', fallback: true }); +}); From 46b8eb45c6e6d27f0fb18760c2c0109a17ad3e30 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sat, 3 Oct 2026 12:47:45 +0000 Subject: [PATCH 2/2] docs(ruvnet-brain): record the 2026-10-03 content-addressed reconcile on ADR-2104 and the skill Co-Authored-By: jjohare --- docs/adr/ADR-2104-direct-control-over-mcp.md | 17 +++++++++++++++++ skills/ruvnet-brain/SKILL.md | 7 +++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/adr/ADR-2104-direct-control-over-mcp.md b/docs/adr/ADR-2104-direct-control-over-mcp.md index 12ef08992..6b315a185 100644 --- a/docs/adr/ADR-2104-direct-control-over-mcp.md +++ b/docs/adr/ADR-2104-direct-control-over-mcp.md @@ -118,3 +118,20 @@ without losing the capability. - **Priority:** P1 — this cycle (Track A item 1; CY-A1, exit test passed 30 Sep) - **Why:** The fail-hard half is proven on the running image. On 30 Sep, `rm /run/agentbox/mcp-hub.json` drove the hub to FATAL after 365 s with no 9720 listener, and restoring the file brought it back (TODO CY-A1; code `d17c557a6`). The inventory is out of date in one row: `ontology-bridge` was retired by ADR-2107 and ADR-2108 (`f5d71b874`), and `[resources.mcp_hub].servers` now lists eight servers (`agentbox.toml:1267`). The port-or-delete follow-on (the `consultant-client` crate, then deleting servers with no call sites) has not started. - **Next:** Amend the inventory to note that the `ontology-bridge` row is closed by ADR-2108, then accept on the CY-A1 exit test. The port half stays `partial`. + +## Note — 2026-10-03: ruvnet-brain corpus ingest follows upstream's content-addressed releases + +No active ADR owns `scripts/ruvnet-brain-ingest.mjs` (its lineage is archived), so this record, the only +active one ruling on the `ruvnet-brain` surface, carries the note. It does not move the order-3 row: the +ingest writes `ruvnet-kb` rows into RuVector, and the MCP server only gained a derived `provenance` block in +`ruvnet_brain_status`. + +- Upstream `stuinfla/ruvnet-brain` tags are now `corpus-sha256-`, with `.zip.sig` and + `corpus-receipt.json`. The ingest takes GitHub's designated `/releases/latest`, because the list API had + put `v4.5.2` ahead of the promoted `corpus-sha256-832bae01…`. It requires `.zip.sha256` to equal the tag + digest and the receipt's archive digest to equal the zip, and it records `sig_present`. It reports + `signature_verified: false` ("no published public key"); no key is guessed. +- Live reconcile, 2026-10-03 12:37–12:46Z: v4.5.1 → `corpus-sha256-832bae01…`, 162,292 chunks (1,518 + embedded, 160,774 unchanged, 450 pruned, 0 failed), receipt `0c210a54…` matched, shipped runtime v4.5.2. +- The ingest does not rebuild the HNSW index. The serial `REINDEX` (ADR-2018) and the recall gate remain an + operator follow-up after each real delta. diff --git a/skills/ruvnet-brain/SKILL.md b/skills/ruvnet-brain/SKILL.md index d4ab4f34b..578af0be1 100644 --- a/skills/ruvnet-brain/SKILL.md +++ b/skills/ruvnet-brain/SKILL.md @@ -23,7 +23,8 @@ depends_on_mcps: ## Where the corpus lives -The ~147k source chunks (186 repos, corpus **v4.2.2-dev**, reconciled 2026-08-21) +The ~162k source chunks (199 repos, corpus +**`corpus-sha256-832bae01…`**, shipped runtime v4.5.2, reconciled 2026-10-03) are rows in **ruvector-postgres** (the shared memory sidecar), namespace **`ruvnet-kb`**, `source_type = ruvnet-brain-ingest`, `memory_type = semantic`. They share the single 384-dim embedding space with every other memory entry, @@ -54,7 +55,9 @@ idempotent: it resolves the newest release that actually carries the zip, fast no-ops when the stamped `corpus_version` matches, and on a real bump embeds only new/changed chunks (content-addressed keys) and prunes rows absent from the new corpus. The 2026-08-14 v3.3.1→v4.0.36 reconcile embedded 6,259 -and pruned 1,807 of 136,439 chunks in ~7 min. +and pruned 1,807 of 136,439 chunks in ~7 min. The 2026-10-03 v4.5.1→`corpus-sha256-832bae01…` +reconcile embedded 1,518, kept 160,774 and pruned 450 (162,292 chunks) in ~9 min, +most of it the 679 MB download. **Content-addressed generations (since 2026-10-03).** Upstream tags are now `corpus-sha256-` (e.g. `corpus-sha256-832bae01…`, shipped