c0mpute edge: mirror c0mpute 0.3.7 site size limits in verified reads

lib/c0mpute-verify.ts

@@ -27,11 +27,19 @@ import { blake3 } from "../vendor/noble-hashes/blake3.js";
 
 /** Files are cut into chunks of this size (site.rs FILE_CHUNK_BYTES). */
 export const FILE_CHUNK_BYTES = 1024 * 1024;
-/** Largest site (site.rs MAX_SITE_BYTES). */
-export const MAX_SITE_BYTES = 1024 * 1024 * 1024;
+const GIB = 1024 * 1024 * 1024;
+/** Largest site: a v2 record, an inline manifest, a parted root (site.rs MAX_SITE_BYTES, 4 TiB). */
+export const MAX_SITE_BYTES = 4 * 1024 * GIB;
+/** Largest site a v1 (free) record may claim (site.rs MAX_FREE_SITE_BYTES). */
+export const MAX_FREE_SITE_BYTES = 64 * GIB;
 const MAX_FILES = 10_000_000;
 const MAX_INLINE_FILES = 10_000;
 const PART_FILES = 4_000;
+/** Chunks per manifest part, besides PART_FILES (site.rs PART_CHUNKS). */
+const PART_CHUNKS = 65_536;
+/** Most parts a v2 root may list (site.rs MAX_PARTS): 2500 + 2 * 64 + 2 = 2630. */
+export const MAX_PARTS = Math.ceil(MAX_FILES / PART_FILES)
+  + 2 * Math.ceil(MAX_SITE_BYTES / FILE_CHUNK_BYTES / PART_CHUNKS) + 2;
 
 const HASH_RE = /^[0-9a-f]{64}$/;
 const KEY_RE = /^[0-9a-f]{64}$/;
@@ -124,7 +132,7 @@ export function verifyRecord(input: unknown, siteId: string): SiteRecord {
   if (!NAME_RE.test(r.name)) fail("bad site name");
   if (!KEY_RE.test(r.owner)) fail("owner is not a hex ed25519 key");
   if (!HASH_RE.test(r.manifest)) fail("manifest is not a hash");
-  if (r.size > MAX_SITE_BYTES) fail("site is over the size cap");
+  if (r.size > (r.v >= 2 ? MAX_SITE_BYTES : MAX_FREE_SITE_BYTES)) fail("site is over the size cap");
   if (!SITE_RE.test(siteId) || r.site_id !== siteId) fail("record is for another site");
   if (siteIdFor(r.owner, r.name) !== r.site_id) fail("site id does not match owner and name");
   if (!ed25519Verify(r.owner, r.sig, signingBytes({ ...r, domains }))) fail("bad record signature");
@@ -196,7 +204,7 @@ export function verifyManifest(bytes: Uint8Array, record: SiteRecord): Manifest
     const parts = m.parts as PartRef[];
     if (!isU64(m.files) || !isU64(m.bytes) || !Array.isArray(parts) || parts.length === 0) fail("manifest root malformed");
     if ((m.files as number) > MAX_FILES || (m.bytes as number) > MAX_SITE_BYTES) fail("manifest root out of bounds");
-    if (parts.length > Math.ceil(MAX_FILES / PART_FILES)) fail("manifest has too many parts");
+    if (parts.length > MAX_PARTS) fail("manifest has too many parts");
     let files = 0, total = 0;
     for (const p of parts) {
       if (!p || typeof p.first !== "string" || typeof p.hash !== "string" || !isU64(p.files) || !isU64(p.bytes)

tests/c0mpute-verify.test.ts

@@ -15,6 +15,9 @@ import { join } from "node:path";
 import {
   ChunkCache,
   FILE_CHUNK_BYTES,
+  MAX_FREE_SITE_BYTES,
+  MAX_PARTS,
+  MAX_SITE_BYTES,
   SeqGuard,
   blake3hex,
   candidates,
@@ -245,6 +248,59 @@ test("v2 parted sites: parts verified by hash, a left-out part is refused, skipp
   assert.throws(() => verifyResolve(s.id, "/zz/foo", lie), /resolved|left out/);
 });
 
+test("size caps mirror site.rs (c0mpute 0.3.7): 4 TiB per site, 64 GiB on a v1 record, 2630 parts", () => {
+  const GIB = 1024 ** 3;
+  assert.equal(MAX_SITE_BYTES, 4 * 1024 * GIB);
+  assert.equal(MAX_FREE_SITE_BYTES, 64 * GIB);
+  assert.equal(MAX_PARTS, 2630);
+
+  const key = newKey();
+  const record = (v: 1 | 2, size: number) => {
+    const base = { site_id: siteIdFor(key.owner, "media"), name: "media", owner: key.owner, manifest: blake3hex("m"), seq: 1, size, domains: [] as string[] };
+    return v === 2
+      ? signRecord(key, { v: 2, ...base, preferred_hosts: ["12D3KooWFTsCjAVPKYnak1H8N7By5UpgeipVScmuAyGeRzzsLwSu"], price_per_gib_month: "0.004" })
+      : signRecord(key, { v: 1, ...base });
+  };
+  const id = siteIdFor(key.owner, "media");
+  assert.equal(verifyRecord(record(2, 3 * GIB), id).size, 3 * GIB);
+  assert.equal(verifyRecord(record(2, MAX_SITE_BYTES), id).size, MAX_SITE_BYTES);
+  assert.throws(() => verifyRecord(record(2, MAX_SITE_BYTES + 1), id), /size cap/);
+  assert.equal(verifyRecord(record(1, 2 * GIB), id).size, 2 * GIB);
+  assert.equal(verifyRecord(record(1, MAX_FREE_SITE_BYTES), id).size, MAX_FREE_SITE_BYTES);
+  assert.throws(() => verifyRecord(record(1, MAX_FREE_SITE_BYTES + 1), id), /size cap/);
+});
+
+test("a v2 site over 1 GiB verifies end to end, with more parts than files alone would need", () => {
+  const GIB = 1024 ** 3;
+  const key = newKey();
+  const name = "media";
+  const index = Buffer.from("<video src=big.mkv>");
+  const size = 3 * GIB + 5;
+  const big: FileEntry = { size, content_type: "video/x-matroska", chunks: Array.from({ length: Math.ceil(size / FILE_CHUNK_BYTES) }, (_, i) => blake3hex(`chunk ${i}`)) };
+  const entries: Record<string, FileEntry> = { "big.mkv": big, "index.html": { size: index.length, content_type: "text/html; charset=utf-8", chunks: [blake3hex(index)] } };
+  const partBytes = Buffer.from(JSON.stringify({ v: 2, files: entries }));
+  const real = { first: "big.mkv", hash: blake3hex(partBytes), files: 2, bytes: size + index.length };
+  // Striped media sites split parts by chunk count too, so a root may list
+  // more parts than MAX_FILES / PART_FILES (2500). Pad with empty-sized parts.
+  const site = (nParts: number, total = real.bytes) => {
+    const refs = [real];
+    for (let i = 1; i < nParts; i++) refs.push({ first: `zz/${String(i).padStart(5, "0")}`, hash: blake3hex(`part ${i}`), files: 1, bytes: 0 });
+    const files = refs.reduce((n, r) => n + r.files, 0);
+    const manifest = Buffer.from(JSON.stringify({ v: 2, name, files, bytes: total, parts: refs }));
+    const rec = signRecord(key, { v: 2, site_id: siteIdFor(key.owner, name), name, owner: key.owner, manifest: blake3hex(manifest), seq: 1, size: total, domains: [], preferred_hosts: ["12D3KooWFTsCjAVPKYnak1H8N7By5UpgeipVScmuAyGeRzzsLwSu"], price_per_gib_month: "0.004" });
+    return { id: rec.site_id, answer: (path: string | null, entry: FileEntry | null): ResolveAnswer => ({ v: 1, record: rec, manifest: manifest.toString("base64"), parts: [partBytes.toString("base64")], path, status: 200, entry }) };
+  };
+  const s = site(1);
+  const r = verifyResolve(s.id, "/big.mkv", s.answer("big.mkv", big));
+  assert.deepEqual([r.status, r.entry!.size, r.entry!.chunks.length], [200, size, 3073]);
+  assert.equal(verifyResolve(s.id, "/", s.answer("index.html", entries["index.html"])).path, "index.html");
+
+  const wide = site(MAX_PARTS);
+  assert.equal(verifyResolve(wide.id, "/big.mkv", wide.answer("big.mkv", big)).entry!.size, size);
+  const tooWide = site(MAX_PARTS + 1);
+  assert.throws(() => verifyResolve(tooWide.id, "/big.mkv", tooWide.answer("big.mkv", big)), /too many parts/);
+});
+
 test("chunks: hash and length checked", () => {
   const body = Buffer.alloc(FILE_CHUNK_BYTES + 10, 7);
   const s = makeSite({ "big.bin": body });