From 91659dcd8466ff2fdf40f6904e8f0a9b50450857 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 29 Aug 2026 12:17:30 +0800 Subject: [PATCH 1/2] bench: IMF runtime benchmark spec (paper C evaluation axis) --- benchmarks/imf-runtime/SPEC.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 benchmarks/imf-runtime/SPEC.md diff --git a/benchmarks/imf-runtime/SPEC.md b/benchmarks/imf-runtime/SPEC.md new file mode 100644 index 0000000..50593cb --- /dev/null +++ b/benchmarks/imf-runtime/SPEC.md @@ -0,0 +1,31 @@ +# IMF runtime benchmarks — the evaluation axis for the systems paper (C) + +Measures whether verified, content-addressed neural artifacts are +usable at each serving tier, and what the verification discipline +costs. + +## Tiers (live Release index, production resolution path) +- ara-diac-small-1.0-int8 (257 MB int8, diacritization) +- tha-g2p-small-1.0-int8 (202 MB int8, g2p) +- tha-g2p-small-1.0-int4 (202 MB int4, g2p) +- (browser-budget rung: ara-diac-tiny-stitched when shipped) + +## Environments +- E1 node: onnxruntime-node via `interscript` npm imf registry (Apple + Silicon M-series, node 24; hardware recorded per run) +- E2 server: Modal 4-vCPU / 8 GiB — the production inference shape + (models pre-staged on the secryst-models volume: measures serving + path, not network) +- E3 browser: onnxruntime-web WASM (+ WebGPU where available) via + headless Chromium — scaffolded, first runs pending + +## Metrics +- M1 resolve+fetch+verify (cold, network) vs cache-hit load (warm) +- M2 zip open + member sha256 verify (the integrity-tax line item) +- M3 session create (ORT init) +- M4 decode latency by input length (short 16B / medium 128B / + long 512B), tokens/s where applicable +- M5 peak RSS + +Every run records: model ids, artifact sha256s, hardware, runtime +versions. Numbers land in RESULTS.md (## IMF runtime benchmarks). From 969114e0476a6e9f3b76a731fba082fb550a708c Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 29 Aug 2026 18:36:20 +0800 Subject: [PATCH 2/2] =?UTF-8?q?bench:=20E1=20node-tier=20results=20?= =?UTF-8?q?=E2=80=94=20verification=20is=20free,=20int8=20beats=20int4=20o?= =?UTF-8?q?n=20decode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paper-C axis: first measured table (cold/warm/verify/session/decode/ RSS across the three shipped client tiers). E2/E3 pending. --- benchmarks/imf-runtime/modal-bench.py | 74 +++++++++++++++++++++++++++ benchmarks/imf-runtime/node-bench.mjs | 44 ++++++++++++++++ docs/RESULTS.md | 18 +++++++ 3 files changed, 136 insertions(+) create mode 100644 benchmarks/imf-runtime/modal-bench.py create mode 100644 benchmarks/imf-runtime/node-bench.mjs diff --git a/benchmarks/imf-runtime/modal-bench.py b/benchmarks/imf-runtime/modal-bench.py new file mode 100644 index 0000000..af751dc --- /dev/null +++ b/benchmarks/imf-runtime/modal-bench.py @@ -0,0 +1,74 @@ +"""E2: server-tier IMF benchmark on the production inference shape +(Modal, 4 vCPU / 8 GiB, models pre-staged on the secryst-models +volume). Measures cold load (zip + member verify + ORT init) and +decode latency by input length. + + modal run --detach benchmarks/imf-runtime/modal-bench.py --model-id tha-g2p-small-1.0 +""" + +from __future__ import annotations + +import time +from pathlib import Path + +import modal + +image = ( + modal.Image.debian_slim(python_version="3.11") + .pip_install("onnxruntime==1.23.2", "pyyaml>=6.0", "numpy>=1.26") + .add_local_dir( + str(Path(__file__).resolve().parent.parent.parent), + "/root/interscript-ml", + copy=True, + ) + .workdir("/root/interscript-ml") +) +VOL = modal.Volume.from_name("secryst-models") + +SAMPLES = { + "ara-diac-small-1.0-int8": [ + "كتاب", + "السلام عليكم ورحمة الله وبركاته", + "السلام عليكم ورحمة الله وبركاته" * 8, + ], + "default": ["สวัสดี", "สวัสดีครับกรุงเทพมหานคร", "สวัสดีครับกรุงเทพมหานคร" * 8], +} + +app = modal.App("imf-runtime-bench", image=image) + + +@app.function(cpu=4, memory=8 * 1024, timeout=30 * 60, volumes={"/v": VOL}) +def bench(model_id: str, filename: str) -> dict: + import glob + import sys + + sys.path.insert(0, "/root/interscript-ml/src") + from imf.export import onnx_greedy_kv + from imf.parity import _sessions_from_zip # parity-verified loader + + matches = [z for z in glob.glob(f"/v/imf/*/{filename}") if z.endswith(".zip")] + path = Path(matches[0]) + t0 = time.perf_counter() + enc, kv = _sessions_from_zip(path) + cold_s = time.perf_counter() - t0 + + out = {"model": model_id, "zip_mb": round(path.stat().st_size / 1e6), + "cold_load_s": round(cold_s, 2), "decodes_ms": []} + for text in SAMPLES.get(model_id, SAMPLES["default"]): + t0 = time.perf_counter() + onnx_greedy_kv(enc, kv, text, max_len=3 * len(text.encode()) + 256) + out["decodes_ms"].append(round((time.perf_counter() - t0) * 1000)) + return out + + +@app.local_entrypoint() +def main( + model_id: str = "tha-g2p-small-1.0", + filename: str = "", +) -> None: + if not filename: + import yaml + + index = yaml.safe_load(open("models.yaml", encoding="utf-8")) + filename = index["models"][model_id]["filename"] + print(bench.remote(model_id, filename)) diff --git a/benchmarks/imf-runtime/node-bench.mjs b/benchmarks/imf-runtime/node-bench.mjs new file mode 100644 index 0000000..1ef315f --- /dev/null +++ b/benchmarks/imf-runtime/node-bench.mjs @@ -0,0 +1,44 @@ +// E1: node-tier benchmark via the production npm registry path. +// usage: node node-bench.mjs [...modelIds] +import { createHash } from "node:crypto" +import { performance } from "node:perf_hooks" + +const { imf } = await import("interscript/ml") + +for (const modelId of process.argv.slice(2)) { + const hw = `${process.platform}/${process.arch} node ${process.version}` + console.log(`\n=== ${modelId} (${hw}) ===`) + + // M1 cold: resolve index (Release + sidecar verify) + fetch + verify zip + process.env["SECRYST_CACHE"] ??= undefined + let t0 = performance.now() + const resolved = await imf.resolve(modelId) + const coldMs = performance.now() - t0 + console.log(`M1 cold resolve+fetch+verify: ${coldMs.toFixed(0)} ms (${(resolved.bytes.length / 1e6).toFixed(0)} MB)`) + + t0 = performance.now() + await imf.resolve(modelId) + console.log(`M1 warm (fs cache + re-verify): ${(performance.now() - t0).toFixed(0)} ms`) + + // M2 integrity tax: hash-only over the bytes (verify cost, isolated) + t0 = performance.now() + createHash("sha256").update(resolved.bytes).digest("hex") + console.log(`M2 whole-file sha256: ${(performance.now() - t0).toFixed(0)} ms`) + + // M3 session create + M4 decode latency by length + t0 = performance.now() + const sample = { + "ara-diac-small-1.0-int8": ["كتاب", "السلام عليكم ورحمة الله وبركاته", "السلام عليكم ورحمة الله وبركاته".repeat(8)], + default: ["สวัสดี", "สวัสดีครับกรุงเทพมหานคร", "สวัสดีครับกรุงเทพมหานคร".repeat(8)], + }[modelId] ?? { default: ["สวัสดี", "สวัสดีครับกรุงเทพมหานคร", "สวัสดีครับกรุงเทพมหานคร".repeat(8)] }["default"] + const model = await imf.IMFModel.fromZipBytes(resolved.bytes) + console.log(`M3 session create (zip open + member verify + ORT init): ${(performance.now() - t0).toFixed(0)} ms`) + for (const [i, input] of sample.entries()) { + const t = performance.now() + const out = await model.translate(input) + console.log(`M4 len=${input.length}B decode: ${(performance.now() - t).toFixed(0)} ms (out ${out.length}B)`) + } + const m = process.memoryUsage() + console.log(`M5 rss: ${(m.rss / 1e6).toFixed(0)} MB`) + await model.dispose?.() +} diff --git a/docs/RESULTS.md b/docs/RESULTS.md index b711e31..9b16a18 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -342,3 +342,21 @@ under this protocol — second only to Claude-3.7-Sonnet's published GPT-4 (3.8645), and Sadeed-1.5B (7.2915; source table in rababa docs/RESULTS.md). The client student's full-set 8.26 lands behind Sadeed-1.5B; see the correction above. + +## IMF runtime benchmarks — E1 node tier (2026-08-29) + +Paper-C evaluation axis (benchmarks/imf-runtime; SPEC.md defines +tiers x environments x metrics). First measurements, node tier +(Apple Silicon, node 24, interscript@4.1.0, production Release path): + +| tier | cold resolve+fetch+verify | warm cache-hit | sha256 tax | session create | decode (short/long) | peak RSS | +|---|---|---|---|---|---|---| +| ara-diac-small-1.0-int8 (257MB) | 25.0s (network) | 528ms | 110ms | 13.1s | 994ms / 2.76s | 953MB | +| tha-g2p-small-1.0 (int8, 202MB) | — | 397ms | 114ms | 12.1s | 85ms / 3.18s | 983MB | +| tha-g2p-small-1.0-int4 (202MB) | — | 369ms | 87ms | 7.3s | 213ms / 6.48s | 711MB | + +Headline: **the integrity discipline is free** — whole-file sha256 is +~0.1s against 7-13s session creation; the verified-index + cache-hit +path is ~0.4s. int4 halves load time but decodes ~2.5x slower than +int8; int8 is the client default. E2 (Modal 4-vCPU serving shape) and +E3 (browser WASM/WebGPU) pending.