bench: E1 node-tier results — verification is free, int8 beats int4 on decode

benchmarks/imf-runtime/SPEC.md

@@ -0,0 +1,31 @@
+# IMF runtime benchmarks — the evaluation axis for the systems paper (C)
+
+Measures whether verified, content-addressed neural artifacts are
+usable at each serving tier, and what the verification discipline
+costs.
+
+## Tiers (live Release index, production resolution path)
+- ara-diac-small-1.0-int8  (257 MB int8, diacritization)
+- tha-g2p-small-1.0-int8   (202 MB int8, g2p)
+- tha-g2p-small-1.0-int4   (202 MB int4, g2p)
+- (browser-budget rung: ara-diac-tiny-stitched when shipped)
+
+## Environments
+- E1 node: onnxruntime-node via `interscript` npm imf registry (Apple
+  Silicon M-series, node 24; hardware recorded per run)
+- E2 server: Modal 4-vCPU / 8 GiB — the production inference shape
+  (models pre-staged on the secryst-models volume: measures serving
+  path, not network)
+- E3 browser: onnxruntime-web WASM (+ WebGPU where available) via
+  headless Chromium — scaffolded, first runs pending
+
+## Metrics
+- M1 resolve+fetch+verify (cold, network) vs cache-hit load (warm)
+- M2 zip open + member sha256 verify (the integrity-tax line item)
+- M3 session create (ORT init)
+- M4 decode latency by input length (short 16B / medium 128B /
+  long 512B), tokens/s where applicable
+- M5 peak RSS
+
+Every run records: model ids, artifact sha256s, hardware, runtime
+versions. Numbers land in RESULTS.md (## IMF runtime benchmarks).

benchmarks/imf-runtime/modal-bench.py

@@ -0,0 +1,74 @@
+"""E2: server-tier IMF benchmark on the production inference shape
+(Modal, 4 vCPU / 8 GiB, models pre-staged on the secryst-models
+volume). Measures cold load (zip + member verify + ORT init) and
+decode latency by input length.
+
+    modal run --detach benchmarks/imf-runtime/modal-bench.py --model-id tha-g2p-small-1.0
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+import modal
+
+image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .pip_install("onnxruntime==1.23.2", "pyyaml>=6.0", "numpy>=1.26")
+    .add_local_dir(
+        str(Path(__file__).resolve().parent.parent.parent),
+        "/root/interscript-ml",
+        copy=True,
+    )
+    .workdir("/root/interscript-ml")
+)
+VOL = modal.Volume.from_name("secryst-models")
+
+SAMPLES = {
+    "ara-diac-small-1.0-int8": [
+        "كتاب",
+        "السلام عليكم ورحمة الله وبركاته",
+        "السلام عليكم ورحمة الله وبركاته" * 8,
+    ],
+    "default": ["สวัสดี", "สวัสดีครับกรุงเทพมหานคร", "สวัสดีครับกรุงเทพมหานคร" * 8],
+}
+
+app = modal.App("imf-runtime-bench", image=image)
+
+
+@app.function(cpu=4, memory=8 * 1024, timeout=30 * 60, volumes={"/v": VOL})
+def bench(model_id: str, filename: str) -> dict:
+    import glob
+    import sys
+
+    sys.path.insert(0, "/root/interscript-ml/src")
+    from imf.export import onnx_greedy_kv
+    from imf.parity import _sessions_from_zip  # parity-verified loader
+
+    matches = [z for z in glob.glob(f"/v/imf/*/{filename}") if z.endswith(".zip")]
+    path = Path(matches[0])
+    t0 = time.perf_counter()
+    enc, kv = _sessions_from_zip(path)
+    cold_s = time.perf_counter() - t0
+
+    out = {"model": model_id, "zip_mb": round(path.stat().st_size / 1e6),
+           "cold_load_s": round(cold_s, 2), "decodes_ms": []}
+    for text in SAMPLES.get(model_id, SAMPLES["default"]):
+        t0 = time.perf_counter()
+        onnx_greedy_kv(enc, kv, text, max_len=3 * len(text.encode()) + 256)
+        out["decodes_ms"].append(round((time.perf_counter() - t0) * 1000))
+    return out
+
+
+@app.local_entrypoint()
+def main(
+    model_id: str = "tha-g2p-small-1.0",
+    filename: str = "",
+) -> None:
+    if not filename:
+        import yaml
+
+        index = yaml.safe_load(open("models.yaml", encoding="utf-8"))
+        filename = index["models"][model_id]["filename"]
+    print(bench.remote(model_id, filename))

benchmarks/imf-runtime/node-bench.mjs

@@ -0,0 +1,44 @@
+// E1: node-tier benchmark via the production npm registry path.
+// usage: node node-bench.mjs <modelId> [...modelIds]
+import { createHash } from "node:crypto"
+import { performance } from "node:perf_hooks"
+
+const { imf } = await import("interscript/ml")
+
+for (const modelId of process.argv.slice(2)) {
+  const hw = `${process.platform}/${process.arch} node ${process.version}`
+  console.log(`\n=== ${modelId} (${hw}) ===`)
+
+  // M1 cold: resolve index (Release + sidecar verify) + fetch + verify zip
+  process.env["SECRYST_CACHE"] ??= undefined
+  let t0 = performance.now()
+  const resolved = await imf.resolve(modelId)
+  const coldMs = performance.now() - t0
+  console.log(`M1 cold resolve+fetch+verify: ${coldMs.toFixed(0)} ms (${(resolved.bytes.length / 1e6).toFixed(0)} MB)`)
+
+  t0 = performance.now()
+  await imf.resolve(modelId)
+  console.log(`M1 warm (fs cache + re-verify): ${(performance.now() - t0).toFixed(0)} ms`)
+
+  // M2 integrity tax: hash-only over the bytes (verify cost, isolated)
+  t0 = performance.now()
+  createHash("sha256").update(resolved.bytes).digest("hex")
+  console.log(`M2 whole-file sha256: ${(performance.now() - t0).toFixed(0)} ms`)
+
+  // M3 session create + M4 decode latency by length
+  t0 = performance.now()
+  const sample = {
+    "ara-diac-small-1.0-int8": ["كتاب", "السلام عليكم ورحمة الله وبركاته", "السلام عليكم ورحمة الله وبركاته".repeat(8)],
+    default: ["สวัสดี", "สวัสดีครับกรุงเทพมหานคร", "สวัสดีครับกรุงเทพมหานคร".repeat(8)],
+  }[modelId] ?? { default: ["สวัสดี", "สวัสดีครับกรุงเทพมหานคร", "สวัสดีครับกรุงเทพมหานคร".repeat(8)] }["default"]
+  const model = await imf.IMFModel.fromZipBytes(resolved.bytes)
+  console.log(`M3 session create (zip open + member verify + ORT init): ${(performance.now() - t0).toFixed(0)} ms`)
+  for (const [i, input] of sample.entries()) {
+    const t = performance.now()
+    const out = await model.translate(input)
+    console.log(`M4 len=${input.length}B decode: ${(performance.now() - t).toFixed(0)} ms (out ${out.length}B)`)
+  }
+  const m = process.memoryUsage()
+  console.log(`M5 rss: ${(m.rss / 1e6).toFixed(0)} MB`)
+  await model.dispose?.()
+}

docs/RESULTS.md

@@ -342,3 +342,21 @@ under this protocol — second only to Claude-3.7-Sonnet's published
 GPT-4 (3.8645), and Sadeed-1.5B (7.2915; source table in rababa
 docs/RESULTS.md). The client student's full-set 8.26 lands behind
 Sadeed-1.5B; see the correction above.
+
+## IMF runtime benchmarks — E1 node tier (2026-08-29)
+
+Paper-C evaluation axis (benchmarks/imf-runtime; SPEC.md defines
+tiers x environments x metrics). First measurements, node tier
+(Apple Silicon, node 24, interscript@4.1.0, production Release path):
+
+| tier | cold resolve+fetch+verify | warm cache-hit | sha256 tax | session create | decode (short/long) | peak RSS |
+|---|---|---|---|---|---|---|
+| ara-diac-small-1.0-int8 (257MB) | 25.0s (network) | 528ms | 110ms | 13.1s | 994ms / 2.76s | 953MB |
+| tha-g2p-small-1.0 (int8, 202MB) | — | 397ms | 114ms | 12.1s | 85ms / 3.18s | 983MB |
+| tha-g2p-small-1.0-int4 (202MB) | — | 369ms | 87ms | 7.3s | 213ms / 6.48s | 711MB |
+
+Headline: **the integrity discipline is free** — whole-file sha256 is
+~0.1s against 7-13s session creation; the verified-index + cache-hit
+path is ~0.4s. int4 halves load time but decodes ~2.5x slower than
+int8; int8 is the client default. E2 (Modal 4-vCPU serving shape) and
+E3 (browser WASM/WebGPU) pending.