From f3c2003c5d189c990e0565f4f3c01e4f739813f7 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 21:13:07 -0500 Subject: [PATCH 01/13] chore(deps): marked 18.1.1, on a byte diff of every README and post marked 18.1.1 changes rendered output (blockquote lazy lines, html blocks interrupting paragraphs, backslashes in code spans, tab stops), so it was held a night. All 8 READMEs and all 98 posts (94 from the live content volume, 4 fixtures) render byte-identical through renderMarkdown on 18.1.0 and 18.1.1; five of eight probes taken from the release notes do differ between the two, so the diff could see a change had there been one. package.json, package-lock.json, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 21 +++++++++++++-------- package-lock.json | 8 ++++---- package.json | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5dbbcc6..fd0de13 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -491,14 +491,19 @@ record. There is no run log. Golar's ~2.7 s. -- [ ] marked 18.1.1 (published 2026-10-12 12:28 UTC; it renders every README - and post). Held a night: it changes output, not just speed — a - blockquote ends before a lazy line after a heading, hr or html block; - indented and type 3–5 html blocks may interrupt a paragraph; a backslash - in a code span no longer hides emphasis; list continuation respects tab - stops. Its `walkTokens` fix (quadratic → linear, marked#4120) does not - apply here: `shared/markdown/render.ts` registers no `walkTokens`. Take - it with a byte diff of all eight READMEs and every post, before and after. +- [x] marked 18.1.1 (published 2026-10-12 12:28 UTC; it renders every README + and post), landed 2026-10-13 after a night held. It changes output, not + just speed — a blockquote ends before a lazy line after a heading, hr or + html block; indented and type 3–5 html blocks may interrupt a paragraph; + a backslash in a code span no longer hides emphasis; list continuation + respects tab stops. So it went in on a byte diff: all 8 READMEs (as + `absolutize(stripLeadingLogo())` prepares them) and all 98 posts — the + 94 on the live content volume and the 4 fixtures — rendered through + `renderMarkdown` on 18.1.0 and on 18.1.1 are byte-identical. The diff + was shown able to see the change: five of eight probes built from the + release notes render differently between the two. Its `walkTokens` fix + (quadratic → linear, marked#4120) does not apply here: + `shared/markdown/render.ts` registers no `walkTokens`. - [ ] Move the type checker to TypeScript 7 once 7.1 ships its API and vue-tsc diff --git a/package-lock.json b/package-lock.json index 05ba90b..b1808c6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@nuxt/ui": "^4.11.2", "bcryptjs": "^3.0.3", - "marked": "^18.1.0", + "marked": "^18.1.1", "nuxt": "^4.6.1", "shiki": "^4.4.3", "vue": "^3.5.43", @@ -9139,9 +9139,9 @@ } }, "node_modules/marked": { - "version": "18.1.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-18.1.0.tgz", - "integrity": "sha512-PamYXWWWg2nboG3oX5Ffzpy3EFZROqZJK9hSlMmUnekp7TxPs5DcT95vwd46m73/exfKCRVMtgXoPMI4FEJL6w==", + "version": "18.1.1", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.1.1.tgz", + "integrity": "sha512-zEGgrtUG8crk+TrMacJSVABU4DYDDkS8uqgU3ohrn/2fW9C2AEfUrO3m6IhpIpQpl7YcVpiXXC9cSMWDANT3dg==", "license": "MIT", "bin": { "marked": "bin/marked.js" diff --git a/package.json b/package.json index 66ca963..6345846 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "dependencies": { "@nuxt/ui": "^4.11.2", "bcryptjs": "^3.0.3", - "marked": "^18.1.0", + "marked": "^18.1.1", "nuxt": "^4.6.1", "shiki": "^4.4.3", "vue": "^3.5.43", From 8449f84d74e9d405bd2bd913e9b4db8efaf8e83c Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 21:14:17 -0500 Subject: [PATCH 02/13] chore(deps): vitest 5.0.4; Vue 3.5.44 held a night vitest is dev-only and nothing it touches reaches the build: typecheck clean, 276/276 tests, build green, every route's status code identical to the previous build's. Vue 3.5.44 shipped this morning with SSR, hydration and scheduler changes, so it goes on the roadmap with a diff plan instead. package.json, package-lock.json, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 8 +++++ package-lock.json | 76 ++++++++++++++++++++++++++++------------------- package.json | 2 +- 3 files changed, 54 insertions(+), 32 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index fd0de13..fb32a2c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -506,6 +506,14 @@ record. There is no run log. `shared/markdown/render.ts` registers no `walkTokens`. +- [ ] Vue 3.5.44 (published 2026-10-13 07:45 UTC). Held a night, as marked + 18.1.1 was: 24 fixes, several in what this site leans on — SSR + (`server-renderer` propagates async setup errors; `ssr` honours falsy + vnode content overrides), hydration, and the scheduler ("only flush + jobs queued by the root render itself", vuejs/core#15757). Take it with + every route's SSR body diffed against 3.5.43's build (same data, same + snapshot), then `check`, `a11y:browser` and `check:navigation` in both + engines. - [ ] Move the type checker to TypeScript 7 once 7.1 ships its API and vue-tsc adopts it — the 6.0 API is a bridge, not a destination. diff --git a/package-lock.json b/package-lock.json index b1808c6..30efe29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,7 @@ "subset-font": "^2.9.0", "tailwindcss": "^4.3.3", "typescript": "npm:@typescript/typescript6@^6.0.2", - "vitest": "^5.0.2", + "vitest": "^5.0.4", "vue-tsc": "^3.3.13" } }, @@ -4637,22 +4637,22 @@ } }, "node_modules/@vitest/mocker": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.3.tgz", - "integrity": "sha512-T8sWAIbkSyAjkwTcaEc3Iu0o9A27X1/kdXrizhZkGuSKScRQtRzclfAMpOTcGdXCsqxeWlpGy3XjqaW8CpLORg==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.4.tgz", + "integrity": "sha512-tqAlrYWce4iUxhr3ncNohejtCTOcHeE8XWrSNZY3OhPoDn01yeGv+gnaBy64ATp7o6pb03eIbBAjzmibpc9vIQ==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "0.3.31", - "@vitest/spy": "5.0.3", + "@vitest/spy": "5.0.4", "estree-walker": "^3.0.3", - "magic-string": "^1.2.3" + "magic-string": "^1.4.3" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "msw": "^2.4.9", + "msw": "^2.4.9 || ^3.0.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { @@ -4675,9 +4675,9 @@ } }, "node_modules/@vitest/spy": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.3.tgz", - "integrity": "sha512-XhFysQTB8AZ+P4gMi+Lpo99vg2AZi0qKpaB9yXQl37+CaMEAPO3iH/wGVnSyL5MPERiLezpqTVtrR6UZH5GCXg==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.4.tgz", + "integrity": "sha512-MZKk/gSkm2q+ICobMxCpH1mA+cCAWm8WV7UhxlqGjq/sJ3AGc1BqFOs5CRVXxH78wQtme9q/NAAgBquj4AcyMw==", "dev": true, "license": "MIT", "funding": { @@ -5880,9 +5880,9 @@ } }, "node_modules/chai": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", - "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.3.0.tgz", + "integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==", "dev": true, "license": "MIT", "engines": { @@ -12146,9 +12146,9 @@ "license": "MIT" }, "node_modules/tinybench": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.0.tgz", - "integrity": "sha512-78U2TlB2CnVenajOFzf3BKSm0J6oz5L0NV7g32LCPccvYc0lbWvys4d3uUUCS2B1N8PAf2+aekR8i1KbC3HO7Q==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.2.tgz", + "integrity": "sha512-nIZ2N2dFpjBsg2hgitKu7mA+FvsIJuGJOwysRngmVRRmWgR/UyPjwJj8c/fT/dhm8EkHnF5EOogRf+0Z2qrimA==", "dev": true, "license": "MIT", "engines": { @@ -13348,23 +13348,23 @@ } }, "node_modules/vitest": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.3.tgz", - "integrity": "sha512-xMw97S3rjdtj5dkVat7jCsqWBpvchs3RlpQctUqwJD0KkERk40vz2fJ77lDwW/Vzh/pk18eItYAzkodhSes3jQ==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.4.tgz", + "integrity": "sha512-z0WxZRV2Sg48wF0FSgMq3TFG8JS38/Qr96vZVZdX6+Em9raQTvnXQvdTFJe2m3rfyJnEHeHQ/snceIdi8D1pVg==", "dev": true, "license": "MIT", "dependencies": { "@types/chai": "^5.2.2", - "@vitest/mocker": "5.0.3", - "chai": "^6.2.2", + "@vitest/mocker": "5.0.4", + "chai": "^6.3.0", "es-module-lexer": "^2.3.2", "expect-type": "^1.4.0", - "magic-string": "^1.2.3", - "obug": "^2.1.4", + "magic-string": "^1.4.3", + "obug": "^3.0.0", "picomatch": "^4.0.7", - "std-env": "^4.2.0", - "tinybench": "^6.1.4", - "tinyexec": "^1.3.0", + "std-env": "^4.3.0", + "tinybench": "^6.2.2", + "tinyexec": "^1.3.1", "tinyglobby": "^0.2.17", "why-is-node-running": "3.2.1" }, @@ -13381,12 +13381,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "5.0.3", - "@vitest/browser-preview": "5.0.3", + "@vitest/browser-playwright": "5.0.4", + "@vitest/browser-preview": "5.0.4", "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", - "@vitest/coverage-istanbul": "5.0.3", - "@vitest/coverage-v8": "5.0.3", - "@vitest/ui": "5.0.3", + "@vitest/coverage-istanbul": "5.0.4", + "@vitest/coverage-v8": "5.0.4", + "@vitest/ui": "5.0.4", "happy-dom": "*", "jsdom": "*", "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" @@ -13430,6 +13430,20 @@ } } }, + "node_modules/vitest/node_modules/obug": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/obug/-/obug-3.0.0.tgz", + "integrity": "sha512-5vvB5+W7ePv+p3uqxi+RcW1XAzLW0/hxt3/4X4Lc4qHudzOhmBiBwOY6DRob4WnanAEGvNcLjF+KNOufrUoEQw==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/vscode-uri": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.2.0.tgz", diff --git a/package.json b/package.json index 6345846..23c6e02 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "subset-font": "^2.9.0", "tailwindcss": "^4.3.3", "typescript": "npm:@typescript/typescript6@^6.0.2", - "vitest": "^5.0.2", + "vitest": "^5.0.4", "vue-tsc": "^3.3.13" }, "overrides": { From 1ec2ac79b42f5d32d136a3b367c8eec0d2c3837c Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 22:31:22 -0500 Subject: [PATCH 03/13] fix(scroll): a cold load of #heading lands on it, not 4 or 18 px past Two scrolls raced on a fresh load with a fragment. HTML's scroll to the fragment uses behavior auto, so the stylesheet's scroll-behavior: smooth made Chromium's own fragment scroll an animation from the top, while vue-router jumped to the same heading as the app started; when the animation's first step (always 4 or 18 px here) landed after the jump it was added on top. restore-jump.client.ts now holds scroll-behavior: auto for a fresh load with a hash too, as it did for reloads and Back/Forward. Nanna at 390 px, #api-key-invalid, uninstrumented: 4/20 cold loads off at full speed and 19/20 at 6x CPU before; 0/20 and 0/20 after. check:navigation could not see a +4 miss (text still 12 px clear of the masthead), so it now also wants the heading's box on scroll-padding-top's line within 2 px, and takes the cold load at 6x CPU in Chromium via a new page.throttle() in scripts/lib/browser.mjs (a no-op in Firefox). Main's build fails it on 9 cold loads across 5 pages; this build passes 160/160 in Chromium and Firefox. app/plugins/restore-jump.client.ts, scripts/check-navigation.mjs, scripts/lib/browser.mjs, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 52 ++++++++++++++++++++++-------- app/plugins/restore-jump.client.ts | 18 ++++++++--- scripts/check-navigation.mjs | 33 +++++++++++++++++-- scripts/lib/browser.mjs | 10 ++++++ 4 files changed, 92 insertions(+), 21 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index fb32a2c..e104180 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1188,20 +1188,44 @@ record. There is no run log. smooth scrolling back. Chromium, `/` and `/projects` at 1,200 px: 34 scroll events → 1, landing at 1,200 every time; Firefox unchanged (1). A fresh visit, and a page without script, are untouched. -- [ ] A cold load of `about#` can land a couple of pixels short of - the masthead's edge: once in three `check:navigation` runs in Chromium - on 2026-10-12, Nanna at 390 px, `#api-key-invalid` 16,253 px down, its - first line 2 px under the pane (the band is 0–200), so CI can go red on - it. A fresh `navigate` load, which none of that night's restore changes - touch (each returns early unless the load is a reload or Back/Forward). - Over six Chromium runs of that build: four passed all 160, one had this - miss, and one stalled on the same page (Nanna, phone) until its 15-minute - timeout, with the host at load 9 and other agents' browsers running. - Main's build: no such miss in three runs, so it is not shown to predate - the night. Alone, 25 cold loads of that URL landed at 16 or 12 px, never - out of band. Next: the same loop inside the check's own sequence (after - the reload and Back cases on that tab), with its frame trace, on both - builds, and find out what the stalled run was waiting for. +- [x] A cold load of `about#` landed a few pixels past the heading, + sometimes far enough to put its first line behind the masthead (Nanna + at 390 px, `#api-key-invalid`, 2 px under the pane, once in three + `check:navigation` runs in Chromium on 2026-10-12). Two scrolls raced. + HTML's "scroll to the fragment" scrolls the target into view with + behavior `auto`, so the stylesheet's `scroll-behavior: smooth` made + Chromium's own fragment scroll an animation from the top; vue-router + jumped to the same heading as the app started, and when the animation's + first step came after the jump it was added on top of it. With + vue-router's jump held back the browser opened its animation with a step + of 4 or 18 px on 12 of 12 loads, and every miss was exactly +4 or +18. + `app/plugins/restore-jump.client.ts` now holds `scroll-behavior: auto` + on a fresh load with a fragment too, as it already did for reloads and + Back/Forward. Uninstrumented, the same URL: before, 4 of 20 cold loads + off at full speed and 19 of 20 with the CPU slowed 6× (one 21 px behind + the pane); after, 0 of 20 and 0 of 20. The check could not see most of + this: a +4 miss leaves the text 12 px clear, inside the 0–200 band. It + now also wants the heading's box on `scroll-padding-top`'s line (±2 px, + lower only at the end of the page), and takes the cold load with the + CPU slowed 6× in Chromium (`page.throttle()`). Main's build fails it on + 9 cold loads across 5 pages; this one passes 160 of 160 in Chromium + and Firefox. CSSOM View says a new scroll aborts any ongoing smooth + scroll, which is not what the +4/+18 looks like. + + + - [ ] Make a minimal repro outside this site (a long page, `scroll-behavior: + smooth`, a `#fragment`, one instant `scrollTo` to the same target as + the document loads) before anyone reports it to Chromium — + reporting is the **owner's call**, as with the Nuxt one above. + - [ ] Find out what the 2026-10-12 stalled run (Nanna, phone, held until + the 15-minute step timeout) was waiting for. + - [ ] One reload of splimes' open README at 1280 px landed 31 px past + the reader (4,170 → 4,201) in that same run against main's build. + The trace shows the fold restoring 4,201 — the position + `pagehide` saved — so the page had moved by 31 px between the + check's last reading and the unload. Seen once; not seen on this + branch's build in four runs. Next: whether a late README image + above the reader can still move it after `arrive` settles. - [ ] Look at the masthead in a real Firefox and in Tor Browser, scrolled. Headless Firefox drew no blur at all behind it, not even the plain `blur()` the CSS gives every other browser, so its screenshots show body diff --git a/app/plugins/restore-jump.client.ts b/app/plugins/restore-jump.client.ts index a2ef7cb..5d52bc3 100644 --- a/app/plugins/restore-jump.client.ts +++ b/app/plugins/restore-jump.client.ts @@ -8,16 +8,26 @@ * vue-router's restores were already made instant (`router.options.ts`), but * the browser's cannot be asked to be: so on a reload, or Back or Forward to * a document, the page holds `scroll-behavior: auto` until scrolling has been - * still for a moment, then gives the stylesheet's smooth scrolling back. A - * fresh visit is not touched; neither is a page without script, which keeps - * the stylesheet's behaviour throughout. + * still for a moment, then gives the stylesheet's smooth scrolling back. + * + * A fresh visit to a `#fragment` jumps too. Chromium scrolls to the fragment + * itself — smoothly, from the top, under the same stylesheet — while + * vue-router jumps to the same heading as the app starts. The two met in one + * frame, and when the animation's first step landed after the jump it was + * added on top of it: Nanna's about page at 390 px, `#api-key-invalid`, + * landed 4 or 18 px past the heading on 4 of 22 cold loads, the 18 putting + * its first line behind the masthead. The browser alone, with vue-router's + * jump held back, opened its animation with a step of 4 or 18 px every time. + * A visit with no fragment is not touched; neither is a page without + * script, which keeps the stylesheet's behaviour throughout. */ export default defineNuxtPlugin({ name: 'restore-jump', order: -25, setup() { const nav = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming | undefined - if (nav?.type !== 'reload' && nav?.type !== 'back_forward') return + const returning = nav?.type === 'reload' || nav?.type === 'back_forward' + if (!returning && !location.hash) return const root = document.documentElement root.style.setProperty('scroll-behavior', 'auto') diff --git a/scripts/check-navigation.mjs b/scripts/check-navigation.mjs index b71fa70..bdfa641 100644 --- a/scripts/check-navigation.mjs +++ b/scripts/check-navigation.mjs @@ -20,7 +20,8 @@ * - Forward and Back again: the same * - reload: unfolded, same place * - blog tab, then the about tab by its link: folded, at the top - * - a cold load of `about#`: unfolded, the heading at the top + * - a cold load of `about#`, CPU slowed 6× in Chromium: unfolded, + * the heading at the top * - a client navigation to `about#`: the same * - fold it, blog tab, Back: still folded — even with the fragment * - opened before the app hydrates: still open after it has @@ -47,6 +48,8 @@ const SLACK = 4 * once passed at 106 px under a 128 px pane. */ const HEADING_BAND = [0, 200] +/** How far from `scroll-padding-top`'s line a heading may sit: rounding, not a race. */ +const AIM_SLACK = 2 /** The same two widths `check-a11y-browser.mjs` uses. */ const VIEWPORTS = [ @@ -126,7 +129,24 @@ try { const glass = document.querySelector('header .crt-glass').getBoundingClientRect() return Math.round((line ?? el.getBoundingClientRect()).top - glass.bottom) })()` : 'null'}, + // The heading's box, which is what a scroll aligns — its text can sit + // lower inside it — and where the stylesheet's \`scroll-padding-top\` + // puts that box, both from the masthead's edge like \`heading\`; and + // whether the page is as far down as it goes. + box: ${id ? `Math.round((document.getElementById(${JSON.stringify(id)})?.getBoundingClientRect().top ?? -1e9) - document.querySelector('header .crt-glass').getBoundingClientRect().bottom)` : 'null'}, + aim: ${id ? `Math.round(parseFloat(getComputedStyle(document.documentElement).scrollPaddingTop) - document.querySelector('header .crt-glass').getBoundingClientRect().bottom)` : 'null'}, + end: Math.ceil(scrollY + innerHeight) >= document.documentElement.scrollHeight - 1, })`) + /** + * A heading scrolled to has its box where `scroll-padding-top` puts it — + * within a pixel or two, not merely with its text somewhere clear of the + * masthead: the race fixed in `app/plugins/restore-jump.client.ts` most + * often landed 4 px past it, which the 0–200 band alone let through. + * Lower is fine only when the page cannot scroll any further. + */ + const atHeading = (s) => s.open === true + && s.heading >= HEADING_BAND[0] && s.heading <= HEADING_BAND[1] + && (Math.abs(s.box - s.aim) <= AIM_SLACK || (s.end && s.box > s.aim)) for (const slug of slugs) { const about = `/projects/${slug}/about` @@ -217,17 +237,24 @@ try { const hash = `#${encodeURIComponent(id)}` await page.goto('about:blank') + // Slowed, in Chromium: the browser's own scroll to the fragment and + // vue-router's met in one frame, and the animation's first step could + // land on top of the jump — 4 or 18 px past the heading. At full speed + // that was 7 cold loads in 40 and CI red now and then; at 6× it was 14 + // in 20, so this fails reliably if it comes back. + await page.throttle(6) await page.goto(BASE + about + hash) await arrive(about) + await page.throttle(1) s = await state(id) - expect(`a cold load of ${hash} opens it at the heading`, s.open === true && s.heading >= HEADING_BAND[0] && s.heading <= HEADING_BAND[1], s) + expect(`a cold load of ${hash} opens it at the heading`, atHeading(s), s) await click(`a[href="${blog}"]`) await arrive(blog) await page.evaluate(`document.querySelector('#__nuxt').__vue_app__.config.globalProperties.$router.push(${JSON.stringify(about + hash)})`) await arrive(about) s = await state(id) - expect(`a client navigation to ${hash} opens it at the heading`, s.open === true && s.heading >= HEADING_BAND[0] && s.heading <= HEADING_BAND[1], s) + expect(`a client navigation to ${hash} opens it at the heading`, atHeading(s), s) await click('details > summary') await sleep(300) diff --git a/scripts/lib/browser.mjs b/scripts/lib/browser.mjs index 4069353..0705fca 100644 --- a/scripts/lib/browser.mjs +++ b/scripts/lib/browser.mjs @@ -68,6 +68,14 @@ async function chromiumDriver() { const { identifier } = await cdp.send('Page.addScriptToEvaluateOnNewDocument', { source }, sessionId) return () => cdp.send('Page.removeScriptToEvaluateOnNewDocument', { identifier }, sessionId) }, + /** + * Run the page's CPU `rate` times slower (1 is full speed). A race + * that a fast machine wins every time can lose on a busy CI runner; + * slowed, it loses here too. + */ + async throttle(rate) { + await cdp.send('Emulation.setCPUThrottlingRate', { rate }, sessionId) + }, async reload() { const loaded = cdp.once('Page.loadEventFired', sessionId) await cdp.send('Page.reload', {}, sessionId) @@ -134,6 +142,8 @@ async function firefoxDriver() { }) return () => bidi.send('script.removePreloadScript', { script }) }, + /** BiDi has no CPU throttling: Firefox runs at full speed. */ + async throttle() {}, async reload() { await bidi.send('browsingContext.reload', { context, wait: 'complete' }) }, From 4ea706c87df7d26fb8aa0fa7382d3b39c861dfb6 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 22:31:30 -0500 Subject: [PATCH 04/13] fix(checks): a browser request that never answers fails in 180 s, named The CDP and BiDi clients sent requests with no timeout, so a page whose renderer stopped answering held a check until CI's 15-minute step limit with nothing to say what it waited for (check:navigation on Nanna at 390 px, 2026-10-12). Every request now rejects after REQUEST_TIMEOUT_MS (180 s) with its method and the expression or URL it carried. Proved with an evaluate of a promise that never settles: rejected at 180 s in Chromium (Runtime.evaluate) and Firefox (script.evaluate), and the page answered the next call in milliseconds. check:navigation and a11y:browser in both engines ran green under it, with no false fires. scripts/lib/cdp.mjs, scripts/lib/bidi.mjs, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 3 ++- scripts/lib/bidi.mjs | 6 ++++-- scripts/lib/cdp.mjs | 25 +++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index e104180..409c9f4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1218,7 +1218,8 @@ record. There is no run log. the document loads) before anyone reports it to Chromium — reporting is the **owner's call**, as with the Nuxt one above. - [ ] Find out what the 2026-10-12 stalled run (Nanna, phone, held until - the 15-minute step timeout) was waiting for. + the 15-minute step timeout) was waiting for. Every CDP and BiDi + request now fails after 180 s, naming itself, so the next stall says. - [ ] One reload of splimes' open README at 1280 px landed 31 px past the reader (4,170 → 4,201) in that same run against main's build. The trace shows the fold restoring 4,201 — the position diff --git a/scripts/lib/bidi.mjs b/scripts/lib/bidi.mjs index 2a0084e..431c93d 100644 --- a/scripts/lib/bidi.mjs +++ b/scripts/lib/bidi.mjs @@ -16,6 +16,7 @@ import { spawn, execFileSync } from 'node:child_process' import { mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { unanswered } from './cdp.mjs' export function findFirefox() { if (process.env.FIREFOX_PATH) return process.env.FIREFOX_PATH @@ -78,7 +79,8 @@ async function connect(url) { ws.onmessage = (event) => { const msg = JSON.parse(event.data) if (msg.id !== undefined && pending.has(msg.id)) { - const { res, rej } = pending.get(msg.id) + const { res, rej, timer } = pending.get(msg.id) + clearTimeout(timer) pending.delete(msg.id) if (msg.type === 'error') rej(new Error(`${msg.error}: ${msg.message}`)) else res(msg.result) @@ -90,7 +92,7 @@ async function connect(url) { send(method, params = {}) { const msgId = ++id ws.send(JSON.stringify({ id: msgId, method, params })) - return new Promise((res, rej) => pending.set(msgId, { res, rej })) + return new Promise((res, rej) => pending.set(msgId, { res, rej, timer: unanswered(pending, msgId, rej, method, params) })) }, on(listener) { listeners.add(listener) diff --git a/scripts/lib/cdp.mjs b/scripts/lib/cdp.mjs index 89f00ce..d89a8ff 100644 --- a/scripts/lib/cdp.mjs +++ b/scripts/lib/cdp.mjs @@ -72,6 +72,26 @@ async function launch(chrome, profile) { return { proc, url } } +/** + * How long one request may go unanswered. Nothing a check asks for takes + * this long — an axe run over the longest README is seconds — but a page + * whose renderer stops answering used to hold the check until CI's step + * timeout, 15 minutes later, with nothing to say what it had been waiting + * for (Nanna at 390 px, 2026-10-12). Now it fails, naming the request. + */ +export const REQUEST_TIMEOUT_MS = 180_000 + +/** Reject request `msgId` if it is still pending after `REQUEST_TIMEOUT_MS`. */ +export function unanswered(pending, msgId, rej, method, params) { + const timer = setTimeout(() => { + if (!pending.delete(msgId)) return + const what = params?.expression ?? params?.url ?? '' + rej(new Error(`${method} got no answer in ${REQUEST_TIMEOUT_MS / 1000} s${what ? `: ${String(what).replace(/\s+/g, ' ').slice(0, 120)}` : ''}`)) + }, REQUEST_TIMEOUT_MS) + timer.unref?.() + return timer +} + /** The smallest CDP client that works: numbered requests, flat sessions, events by name. */ async function connect(url) { const ws = new WebSocket(url) @@ -90,7 +110,8 @@ async function connect(url) { const msg = JSON.parse(data) for (const l of listeners) l(msg) if (msg.id && pending.has(msg.id)) { - const { res, rej } = pending.get(msg.id) + const { res, rej, timer } = pending.get(msg.id) + clearTimeout(timer) pending.delete(msg.id) if (msg.error) rej(new Error(`${msg.error.message} (${msg.error.code})`)) else res(msg.result) @@ -108,7 +129,7 @@ async function connect(url) { send(method, params = {}, sessionId) { const msgId = ++id ws.send(JSON.stringify({ id: msgId, method, params, sessionId })) - return new Promise((res, rej) => pending.set(msgId, { res, rej })) + return new Promise((res, rej) => pending.set(msgId, { res, rej, timer: unanswered(pending, msgId, rej, method, params) })) }, once(method, sessionId, timeout = 30_000) { return new Promise((res, rej) => { From 0eddd791ba284e745355ea9c635bc0869e236c9a Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 22:39:49 -0500 Subject: [PATCH 05/13] fix(readme): an image the probe misses keeps the snapshot's size A missed size probe is not cached, so the README is served with that image at 0x0 until the next refresh, and cold starts (every deploy) are where misses land. img.shields.io answered in 3.0 s to 1 request in 9 from this host, past the 2 s budget; a local server tonight served splimes with four badges unsized. statedSizes() reads the sizes a rendered README states, and sizeReadmeImages() now takes the snapshot's copy of the same README's sizes as a fallback for any image the live probe misses. A live answer still wins. Cold-started behind a CONNECT proxy refusing img.shields.io: main's build served 18 README images unsized across 5 about pages, this one 0. 279 tests, typecheck clean, build green, routes unchanged, check clean. shared/markdown/image-size.ts, server/utils/readmeImages.ts, server/utils/github.ts, test/image-size.test.ts, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 20 ++++++++++++++++++-- server/utils/github.ts | 3 ++- server/utils/readmeImages.ts | 12 ++++++++++-- shared/markdown/image-size.ts | 20 ++++++++++++++++++++ test/image-size.test.ts | 23 ++++++++++++++++++++++- 5 files changed, 72 insertions(+), 6 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 409c9f4..f822a31 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -737,6 +737,20 @@ record. There is no run log. Skidbladnir's first render 0.88–1.04 s on main, 1.02–1.47 s here (three runs each); warm renders unchanged at ~9 ms. The server's probe gives up after 2 s, so a hanging image host costs a cold render at most that. +- [x] …and they keep it when a probe misses. A miss is not cached, so the + next refresh retries — but until it does the README is served with that + image at 0 × 0, and a cold start, which is every deploy, is where misses + land. img.shields.io took 3.0 s to answer 1 request in 9 from this host + on 2026-10-13, past the server's 2 s budget, and a local server that + night was serving splimes with four badges unsized (the badge row + rebuilt itself sideways as they loaded; everything below moved 2.3 px). + The snapshot already holds every image's last measured size, so a miss + now falls back to it: `statedSizes()` (`shared/markdown/image-size.ts`) + reads them out of the snapshot's copy of the same README, and + `sizeReadmeImages` takes them as `known`. A live answer still wins. + Cold-started behind a proxy that refuses img.shields.io: main's build + served 18 README images unsized across five about pages (splimes 4, + WeftDB 5, ArtiQwest 4, onyums 3, Skidbladnir 2), this one 0. - [x] A folded README costs no third-party request. Its images had no `loading` attribute, and an eager image inside a closed `
` is fetched anyway, so every visit to the Skidbladnir page fetched two screenshots from @@ -1225,8 +1239,10 @@ record. There is no run log. The trace shows the fold restoring 4,201 — the position `pagehide` saved — so the page had moved by 31 px between the check's last reading and the unload. Seen once; not seen on this - branch's build in four runs. Next: whether a late README image - above the reader can still move it after `arrive` settles. + branch's build in four runs. Not the unsized shields.io badges + that night's local server served (fixed below): they move the + page 2.3 px, not 31. Next: whether anything else above the reader + still moves it after `arrive` settles. - [ ] Look at the masthead in a real Firefox and in Tor Browser, scrolled. Headless Firefox drew no blur at all behind it, not even the plain `blur()` the CSS gives every other browser, so its screenshots show body diff --git a/server/utils/github.ts b/server/utils/github.ts index 8417202..5f85978 100644 --- a/server/utils/github.ts +++ b/server/utils/github.ts @@ -21,6 +21,7 @@ import { ConditionalCache } from '~~/shared/github/conditional' import { projects, type Project } from '~~/data/projects' import type { DownloadRelease, EnrichedProject, Release, RepoMeta } from '~~/shared/types/project' import { absolutize, stripLeadingLogo } from '~~/shared/markdown/readme' +import { statedSizes } from '~~/shared/markdown/image-size' import { normaliseReleases, pickDownload, pickLatest } from '~~/shared/github/releases' import { refreshPolicy } from '~~/shared/github/budget' import { paced } from '~~/shared/net/pace' @@ -211,7 +212,7 @@ async function fetchRepo(project: Project): Promise { if (readme.status === 'fulfilled' && typeof readme.value === 'string') { out.readmeHtml = await sizeReadmeImages(await renderMarkdown( absolutize(stripLeadingLogo(readme.value), ORG, repo, defaultBranch), - )) + ), statedSizes(snapshotRepos[repo]?.readmeHtml ?? '')) } else { incomplete.push('readme') diff --git a/server/utils/readmeImages.ts b/server/utils/readmeImages.ts index 75cd08f..3065f36 100644 --- a/server/utils/readmeImages.ts +++ b/server/utils/readmeImages.ts @@ -29,13 +29,21 @@ const cachedImageSize = defineCachedFunction( * A rendered README with its images' sizes written in, so the page reserves * their space; see `shared/markdown/image-size.ts`. Never fails — an image it * cannot measure keeps the markup it had. + * + * `known` is what an earlier render measured (the snapshot's copy of the same + * README), used for any image the probe misses this time. A miss is not rare: + * img.shields.io took 3.0 s to answer 1 request in 9 from here on 2026-10-13, + * past the 2 s above, and a miss is not cached — so a cold start, which is + * every deploy, could serve splimes' README with four of its badges at 0 × 0 + * until they loaded: the badge row rebuilt itself sideways, and everything + * below it moved 2.3 px. A live answer still wins: it is the newer one. */ -export async function sizeReadmeImages(html: string): Promise { +export async function sizeReadmeImages(html: string, known: ReadonlyMap = new Map()): Promise { const srcs = unsizedImages(html) if (!srcs.length) return html const sizes = new Map() await Promise.all(srcs.map(async (src) => { - const size = await cachedImageSize(src).catch(() => null) + const size = await cachedImageSize(src).catch(() => null) ?? known.get(src) if (size) sizes.set(src, size) })) return sizeImages(html, sizes) diff --git a/shared/markdown/image-size.ts b/shared/markdown/image-size.ts index 30d0459..943a5f4 100644 --- a/shared/markdown/image-size.ts +++ b/shared/markdown/image-size.ts @@ -108,6 +108,26 @@ export function unsizedImages(html: string): string[] { return [...out] } +/** + * The sizes a rendered README already states, by `src`: every `` with a + * numeric `width` and `height`. Read from the snapshot's copy of a README, it + * is the last size each image was measured at — the shape a live render falls + * back to when a probe misses (see `server/utils/readmeImages.ts`). Where an + * author gave a width and the probe the height, the pair still has the + * image's shape, which is all `sizeImages` takes from it. + */ +export function statedSizes(html: string): Map { + const out = new Map() + for (const [, attrs] of html.matchAll(IMG)) { + const a = attrs! + const width = Number(/\swidth="(\d+)"/i.exec(a)?.[1]) + const height = Number(/\sheight="(\d+)"/i.exec(a)?.[1]) + const src = /\ssrc="([^"]+)"/i.exec(a)?.[1] + if (src && width > 0 && height > 0) out.set(decodeEntities(src), { width, height }) + } + return out +} + /** * Write the sizes in. An image with neither dimension gets both; one with only * a `width` (a README's ``) gets the `height` that keeps its diff --git a/test/image-size.test.ts b/test/image-size.test.ts index 7f78535..04a869b 100644 --- a/test/image-size.test.ts +++ b/test/image-size.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { PROBE_BYTES, measureImage, sizeImages, unsizedImages } from '~~/shared/markdown/image-size' +import { PROBE_BYTES, measureImage, sizeImages, statedSizes, unsizedImages } from '~~/shared/markdown/image-size' import { probeImageSize, probeable } from '~~/shared/markdown/image-probe' const bytes = (...parts: (number[] | string)[]) => @@ -148,3 +148,24 @@ describe('probeImageSize', () => { expect(await probeImageSize('https://example.com/x.png', 1000, never)).toBeNull() }) }) + +describe('statedSizes', () => { + it('reads every image that states both dimensions, by decoded src', () => { + const html = '' + expect([...statedSizes(html)]).toEqual([ + ['https://img.shields.io/crates/v/x.svg', { width: 102, height: 20 }], + ['https://x/c.svg?a=1&b=2', { width: 420, height: 173 }], + ]) + }) + + it('skips units, percentages and zeros', () => { + expect(statedSizes('').size).toBe(0) + }) + + it('gives back the shape a live render needs, even where the author set the width', () => { + // The snapshot's copy of ``, sized by an earlier probe. + const known = statedSizes('') + expect(sizeImages('', known)).toBe('') + expect(sizeImages('', known)).toBe('') + }) +}) From 17b45bbb8919beec798623412c109414bc28da75 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 22:40:45 -0500 Subject: [PATCH 06/13] fix(sync): an image the probe misses keeps the previous snapshot's size npm run sync wrote a README image unsized into the snapshot whenever its size probe missed, which takes away the very fallback the server now uses for its own misses. withImageSizes() keeps the previous snapshot's size for such an image (statedSizes) and logs how many it kept. Behind a CONNECT proxy refusing img.shields.io: main's script wrote 22 unsized images into data/projects.generated.json; this one wrote 0, keeping 22 sizes across six READMEs. (Snapshot restored afterwards; the refresh for this run is a separate commit.) scripts/fetch-projects.mjs, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 6 ++++++ scripts/fetch-projects.mjs | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index f822a31..5bc3c96 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -751,6 +751,12 @@ record. There is no run log. Cold-started behind a proxy that refuses img.shields.io: main's build served 18 README images unsized across five about pages (splimes 4, WeftDB 5, ArtiQwest 4, onyums 3, Skidbladnir 2), this one 0. + `npm run sync` had the same hole one level down, where it matters more: + a miss there writes the image into the snapshot unsized, taking the + fallback away. It now keeps the previous snapshot's size and says so + (`· splimes: 4 image size(s) kept from the previous snapshot`). Behind + the same proxy, main's script wrote 22 unsized images into the + snapshot; this one wrote 0. - [x] A folded README costs no third-party request. Its images had no `loading` attribute, and an eager image inside a closed `
` is fetched anyway, so every visit to the Skidbladnir page fetched two screenshots from diff --git a/scripts/fetch-projects.mjs b/scripts/fetch-projects.mjs index 70448d7..ed0f263 100644 --- a/scripts/fetch-projects.mjs +++ b/scripts/fetch-projects.mjs @@ -23,7 +23,7 @@ import { projects } from '../data/projects.ts' // page when it takes over. The copied renderer had no highlighter, so it did. import { absolutize, stripLeadingLogo } from '../shared/markdown/readme.ts' import { renderMarkdown } from '../shared/markdown/render.ts' -import { sizeImages, unsizedImages } from '../shared/markdown/image-size.ts' +import { sizeImages, statedSizes, unsizedImages } from '../shared/markdown/image-size.ts' import { probeImageSize } from '../shared/markdown/image-probe.ts' import { paced } from '../shared/net/pace.ts' import { normaliseReleases, pickDownload, pickLatest } from '../shared/github/releases.ts' @@ -32,14 +32,22 @@ import { carryMissing } from '../shared/github/carry.ts' /** * The same image sizing the server does (`server/utils/readmeImages.ts`), so a * page served from the snapshot reserves its README images' space as a live - * one does. Uncached: this runs once per sync. + * one does. Uncached: this runs once per sync. An image the probe misses keeps + * the size the previous snapshot measured (`known`), as on the server: the + * snapshot is that fallback, and a miss written into it would take it away. */ -async function withImageSizes(html) { +async function withImageSizes(html, repo, known = new Map()) { const sizes = new Map() + let kept = 0 await Promise.all(unsizedImages(html).map(async (src) => { - const size = await probeImageSize(src) + let size = await probeImageSize(src) + if (!size && known.has(src)) { + size = known.get(src) + kept++ + } if (size) sizes.set(src, size) })) + if (kept) console.warn(` · ${repo}: ${kept} image size(s) kept from the previous snapshot`) return sizeImages(html, sizes) } @@ -112,7 +120,7 @@ async function fetchRepo(project) { 'application/vnd.github.raw', ) const prepared = absolutize(stripLeadingLogo(md), ORG, repo, out.defaultBranch) - out.readmeHtml = await withImageSizes(await renderMarkdown(prepared)) + out.readmeHtml = await withImageSizes(await renderMarkdown(prepared), repo, statedSizes(snapshot.repos?.[repo]?.readmeHtml ?? '')) } catch { out.readmeHtml = null From 2780a4d085bf5ede80e0ff2c0e7bd30038d2e4a7 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 22:42:23 -0500 Subject: [PATCH 07/13] feat(health): count how README images were sized /healthz and the status page carry readmeImages: per image per render, measured by a probe (or the day's cached answer), kept at the snapshot's size because the probe missed, or unsized. The fallback landed earlier tonight hides misses from the page; this says whether production has any. With img.shields.io refused from cold: 6 measured, 22 kept, 0 unsized; without: 28 measured. Typecheck clean, 279 tests, build green, routes unchanged; status page checked at 390 and 1280 px. shared/types/health.ts, server/utils/health.ts, server/utils/readmeImages.ts, app/pages/status.vue, README.md, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- ROADMAP.md | 7 +++++++ app/pages/status.vue | 7 +++++++ server/utils/health.ts | 8 ++++++++ server/utils/readmeImages.ts | 4 +++- shared/types/health.ts | 12 ++++++++++++ 6 files changed, 38 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7f1196d..fa1589c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Links to a heading, and keyboard focus, always stop below it. | `/robots.txt` | Allows everything, points at the sitemap | | `/api/projects` | Card-level JSON for every project | | `/api/projects/` | One project, README and release history included | -| `/healthz` | Liveness for the container healthcheck, plus the data source, the onion service's state and uptime | +| `/healthz` | Liveness for the container healthcheck, plus the data source, the onion service's state, uptime, and how many README images were measured, kept at the snapshot's size, or left unsized | Each project page also carries `SoftwareSourceCode` JSON-LD and its own Open Graph card, so a link to it previews as itself rather than as the organization. diff --git a/ROADMAP.md b/ROADMAP.md index 5bc3c96..bf2f3bb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -757,6 +757,13 @@ record. There is no run log. (`· splimes: 4 image size(s) kept from the previous snapshot`). Behind the same proxy, main's script wrote 22 unsized images into the snapshot; this one wrote 0. +- [x] …and a miss is counted where an operator looks. `/healthz` and the + status page now carry `readmeImages` — one count per README image per + render: `measured`, `kept` (the snapshot's size, because the probe + missed) and `unsized` (nobody knew; that image moves the page). With + img.shields.io refused from cold: 6 measured, 22 kept, 0 unsized; + without: 28 measured. Whether production sees misses at all is now a + number, not a guess. - [x] A folded README costs no third-party request. Its images had no `loading` attribute, and an eager image inside a closed `
` is fetched anyway, so every visit to the Skidbladnir page fetched two screenshots from diff --git a/app/pages/status.vue b/app/pages/status.vue index df779bd..03ac76f 100644 --- a/app/pages/status.vue +++ b/app/pages/status.vue @@ -83,6 +83,13 @@ const rows = computed(() => { label: 'github calls', value: `${h.githubCalls.made} since start, ${h.githubCalls.notModified} unchanged (304, free)`, }, + // A README image nobody could measure is a 0 × 0 box until it loads, and + // moves the page when it does; `kept` is a probe that missed, covered by + // the size the snapshot last measured. + { + label: 'readme images', + value: `${h.readmeImages.measured} measured, ${h.readmeImages.kept} kept from the snapshot, ${h.readmeImages.unsized} unsized`, + }, { label: 'onion', value: onionLine(h.onion) }, { label: 'refresh', diff --git a/server/utils/health.ts b/server/utils/health.ts index de7cee6..2a514a5 100644 --- a/server/utils/health.ts +++ b/server/utils/health.ts @@ -83,6 +83,13 @@ export function recordGithubCall(notModified: boolean): void { if (notModified) githubNotModified++ } +const readmeImages = { measured: 0, kept: 0, unsized: 0 } + +/** Called once per README image per render, by `sizeReadmeImages`. */ +export function recordReadmeImage(outcome: keyof typeof readmeImages): void { + readmeImages[outcome]++ +} + /** Called once per resolved repo, by the cached fetch in `github.ts`. */ export function recordSource(source: 'live' | 'stale' | 'snapshot'): void { if (source === 'live') { @@ -210,6 +217,7 @@ export async function health(): Promise { observedAt: new Date(rateLimit.at).toISOString(), }, githubCalls: { made: githubCalls, notModified: githubNotModified }, + readmeImages: { ...readmeImages }, onion, authenticated: githubRefresh.authenticated, refreshSeconds: githubRefresh.ttl, diff --git a/server/utils/readmeImages.ts b/server/utils/readmeImages.ts index 3065f36..5bf4c52 100644 --- a/server/utils/readmeImages.ts +++ b/server/utils/readmeImages.ts @@ -43,7 +43,9 @@ export async function sizeReadmeImages(html: string, known: ReadonlyMap() await Promise.all(srcs.map(async (src) => { - const size = await cachedImageSize(src).catch(() => null) ?? known.get(src) + const measured = await cachedImageSize(src).catch(() => null) + const size = measured ?? known.get(src) + recordReadmeImage(measured ? 'measured' : size ? 'kept' : 'unsized') if (size) sizes.set(src, size) })) return sizeImages(html, sizes) diff --git a/shared/types/health.ts b/shared/types/health.ts index 7d8e77f..d71369a 100644 --- a/shared/types/health.ts +++ b/shared/types/health.ts @@ -57,6 +57,18 @@ export interface Health { made: number notModified: number } + /** + * README images sized since this process started, one per image per render: + * `measured` by a probe of the file (or the day's cached answer), `kept` at + * the size the snapshot last measured because the probe missed, `unsized` + * when neither knew — that image is a 0 × 0 box until it loads, and moves + * the page when it does. See `server/utils/readmeImages.ts`. + */ + readmeImages: { + measured: number + kept: number + unsized: number + } /** * The onion service, judged by the gateway's own self-fetch over Tor. * `off` where no gateway runs (outside the image, or `ONION_ENABLED=0`); From bda734320df5df04ff024cc7d1affeb98350c6d3 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 23:04:08 -0500 Subject: [PATCH 08/13] docs(repro): Chromium's smooth fragment scroll adds onto an instant jump A 1.4 KB page that reproduces, outside the site, the race fixed in restore-jump.client.ts: scroll-behavior: smooth, a #target 16,000 px down, and one instant scrollTo to the same place shortly after load. Chromium 152.0.7977.82 at 390 px, 30 cold loads per case: a jump 10 ms after load lands 4 or 13 px past on 30/30, 20 ms after lands 13 or 24 px past on 30/30; with scroll-behavior: auto, on target 60/60; with no jump, on target 30/30. Reporting it upstream is the owner's call. docs/repro/smooth-fragment-overshoot.html, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 14 +++++++--- docs/repro/smooth-fragment-overshoot.html | 33 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 docs/repro/smooth-fragment-overshoot.html diff --git a/ROADMAP.md b/ROADMAP.md index bf2f3bb..19e45bb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1240,10 +1240,16 @@ record. There is no run log. scroll, which is not what the +4/+18 looks like. - - [ ] Make a minimal repro outside this site (a long page, `scroll-behavior: - smooth`, a `#fragment`, one instant `scrollTo` to the same target as - the document loads) before anyone reports it to Chromium — - reporting is the **owner's call**, as with the Nuxt one above. + - [x] A minimal repro outside this site: + `docs/repro/smooth-fragment-overshoot.html` (1.4 KB, no + dependencies): a 16,000 px spacer, `scroll-behavior: smooth`, a + `#target`, and one instant `scrollTo` to that same place 10 or + 20 ms after `load`. Chromium 152.0.7977.82 at 390 px, 30 cold + loads each: `?when=ms10` lands 4 or 13 px past the target on 30 of + 30, `ms20` 13 or 24 px past on 30 of 30; with `&smooth=0`, on it 60 + of 60; with no `scrollTo` at all, on it 30 of 30. + - [ ] Report it to Chromium with that file — **owner's call**, as with + the Nuxt one above, since it speaks for the organization. - [ ] Find out what the 2026-10-12 stalled run (Nanna, phone, held until the 15-minute step timeout) was waiting for. Every CDP and BiDi request now fails after 180 s, naming itself, so the next stall says. diff --git a/docs/repro/smooth-fragment-overshoot.html b/docs/repro/smooth-fragment-overshoot.html new file mode 100644 index 0000000..07eb0be --- /dev/null +++ b/docs/repro/smooth-fragment-overshoot.html @@ -0,0 +1,33 @@ + + + + + +smooth fragment scroll + instant scrollTo + + + + +
+

Target

+
+ + + From b02913a95c5ad02fa6c5c2f519c4009b4501901c Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 23:06:59 -0500 Subject: [PATCH 09/13] docs(roadmap): Firefox likely drops the masthead blur entirely, with the fix Headless Firefox draws no backdrop-filter at all, plain blur included, so the routine cannot settle what a real Firefox shows. But Mozilla bug 1961378 (open) says backdrop-filter is ignored entirely when an SVG url() filter falls back to a blob image, and its duplicate 1972363 names feImage and feDisplacementMap, which the masthead's two filters use. Firefox passes the @supports that hands it that chain. The item now carries the one-condition fix (not (-moz-appearance: none)) to ship once someone has looked in a real Firefox, before and after. ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 19e45bb..3f9f759 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1262,12 +1262,28 @@ record. There is no run log. that night's local server served (fixed below): they move the page 2.3 px, not 31. Next: whether anything else above the reader still moves it after `arrive` settles. -- [ ] Look at the masthead in a real Firefox and in Tor Browser, scrolled. - Headless Firefox drew no blur at all behind it, not even the plain - `blur()` the CSS gives every other browser, so its screenshots show body - text crisp behind the nav links — most likely the headless software - compositor, not the site, but no screenshot this routine can take - settles it. +- [ ] Look at the masthead in a real Firefox and in Tor Browser, scrolled — + **needs a person with a real Firefox window**, because headless Firefox + draws no `backdrop-filter` at all (checked 2026-10-13: not the site's + chain, not a plain `blur(2.5px)`, not `blur(8px)`, with or without + `gfx.webrender.software`/`gfx.webrender.all`), so no screenshot this + routine can take settles it. There is now a specific reason to expect + it is broken, not just unseen: Mozilla's open bug 1961378 says + `backdrop-filter` is ignored *entirely* when an SVG `url()` filter in it + falls back to a blob image, and a duplicate (1972363) names `feImage` + and `feDisplacementMap` — exactly what `#crt-warp` and `#glass-refract` + use (`app/components/CrtFilter.vue`: 4 `feDisplacementMap`, 2 + `feImage`). Firefox passes the `@supports (backdrop-filter: + url("#crt-warp"))` in `main.css`, so it takes that chain, and the + comment there ("Chrome and Firefox can do this") is likely wrong. If a + real Firefox shows body text crisp behind the pane, the fix is one + condition: `@supports (backdrop-filter: url("#crt-warp")) and (not + (-moz-appearance: none))`, which hands Firefox the plain blur every + other browser without the chain gets (`CSS.supports('-moz-appearance', + 'none')` is true in Firefox 155, false in Chromium 152). Look before + and after; Tor Browser is Firefox 140 ESR and gets the same answer. + + - [x] No dead controls without script. Every page is server-rendered and reads fine with script off, which is how Tor Browser's "Safest" level serves the onion service — except the `[copy]` buttons, which were shown From a3c43bb6582bedd7b03d4561c3b99d874a41d9b9 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 23:34:29 -0500 Subject: [PATCH 10/13] fix(checks): a stalled page fails with where it was and whether it spun MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stall last seen 2026-10-12 recurred tonight in 1 of 3 Chromium check:navigation runs: arrive()'s evaluate on Nanna at 1280 px went 180 s unanswered (the new request timeout caught it; before, CI's 15-minute step did). Both stalls were on Nanna, on the last page of a width's pass. A page that throws or stalls is now a failure of that page, not the end of the run: it reports the last four calls made of it and, in Chromium, the busiest renderer's CPU time over one second (busy(), from SystemInfo.getProcessInfo), then the run continues in a fresh tab. On a stand-in page that spins after load: "stalled after goto → goto → evaluate; renderer busy (0.99 CPU s in 1 s)", and the next page checked. The real check: 160/160 in Chromium and in Firefox. scripts/check-navigation.mjs, scripts/lib/browser.mjs, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 22 ++- scripts/check-navigation.mjs | 268 ++++++++++++++++++++--------------- scripts/lib/browser.mjs | 20 +++ 3 files changed, 191 insertions(+), 119 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 3f9f759..50ca338 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -426,6 +426,13 @@ record. There is no run log. `schedule` runs can be delayed under load and, if the load is high enough, dropped. Count the runs that happened, not the slots. + So far (read 2026-10-13 23:00 CDT): four scheduled runs in ~31 h + against six or seven slots, landing 22 min to 4 h 27 min after + theirs. One first try failed — connecting, timed out at 120 s + (10-12 15:25 UTC) — and its second reached the site in 82.6 s; + the other three reached it on the first try in 11.1, 13.1 and + 13.2 s, through a difficulty-18 gate solved in 0.2–0.6 s. Too few + to weigh C Tor on; read again once there are a dozen. - [ ] Move the runtime to distroless — **needs an owner decision, because the size case for it is gone.** Measured 2026-09-27: the same build on `gcr.io/distroless/nodejs24-debian13:nonroot` is 73.0 MB of image content @@ -1252,7 +1259,20 @@ record. There is no run log. the Nuxt one above, since it speaks for the organization. - [ ] Find out what the 2026-10-12 stalled run (Nanna, phone, held until the 15-minute step timeout) was waiting for. Every CDP and BiDi - request now fails after 180 s, naming itself, so the next stall says. + request now fails after 180 s, naming itself — and on 2026-10-13 + it did, in 1 of 3 Chromium runs: `arrive`'s evaluate on Nanna at + 1280 px went 180 s unanswered, though its own loop gives up after + 8 s. Both stalls were on Nanna — the longest README, 25,000 px at + phone width — and both on the last page of their width's pass, + where the tab is oldest (~7 min). A stalled page no longer ends + the run: it fails with the last four calls made of it and, in + Chromium, the busiest renderer's CPU time over a second (`busy()` + in `scripts/lib/browser.mjs`, from `SystemInfo.getProcessInfo`), + and the run goes on in a fresh tab. Shown on a stand-in page that + spins after `load`: "renderer busy (0.99 CPU s in 1 s)", and the + next page checked. Next: the busy/idle answer from the next real + stall — busy points at the page (layout or script over Nanna's + README), idle at timers held back in an old tab. - [ ] One reload of splimes' open README at 1280 px landed 31 px past the reader (4,170 → 4,201) in that same run against main's build. The trace shows the fold restoring 4,201 — the position diff --git a/scripts/check-navigation.mjs b/scripts/check-navigation.mjs index bdfa641..b7e131d 100644 --- a/scripts/check-navigation.mjs +++ b/scripts/check-navigation.mjs @@ -26,7 +26,10 @@ * - fold it, blog tab, Back: still folded — even with the fragment * - opened before the app hydrates: still open after it has * - * And, as in `check-a11y-browser.mjs`, any console error fails it. + * And, as in `check-a11y-browser.mjs`, any console error fails it. A page + * that stops answering fails too, after 180 s, naming the last calls made of + * it and — in Chromium — whether its renderer was busy or idle; the run then + * carries on with the next page in a fresh tab. */ import { sitePages } from './lib/pages.mjs' @@ -92,7 +95,20 @@ let checked = 0 try { for (const { name, ...viewport } of VIEWPORTS) { - const page = await browser.open(viewport) + // The last few things asked of the page, so a stall can say where it was. + const calls = [] + const traced = (raw) => new Proxy(raw, { + get(target, key) { + const value = target[key] + if (typeof value !== 'function') return value + return (...args) => { + calls.push(`${String(key)}(${args.map((a) => JSON.stringify(typeof a === 'string' ? a.replace(/\s+/g, ' ') : a)?.slice(0, 70)).join(', ')})`) + if (calls.length > 4) calls.shift() + return value.apply(target, args) + } + }, + }) + let page = traced(await browser.open(viewport)) const sleep = (ms) => page.evaluate(`new Promise((r) => setTimeout(r, ${ms}))`) /** * Wait for the router to arrive at `path`, then for the page to stop @@ -149,137 +165,153 @@ try { && (Math.abs(s.box - s.aim) <= AIM_SLACK || (s.end && s.box > s.aim)) for (const slug of slugs) { - const about = `/projects/${slug}/about` - const blog = `/projects/${slug}/blog` - const problems = [] - const expect = (what, ok, got) => { - checked++ - if (!ok) problems.push(`${what} — got ${JSON.stringify(got)}`) - } + try { + const about = `/projects/${slug}/about` + const blog = `/projects/${slug}/blog` + const problems = [] + const expect = (what, ok, got) => { + checked++ + if (!ok) problems.push(`${what} — got ${JSON.stringify(got)}`) + } - await page.goto('about:blank') - await page.goto(BASE + about) - await arrive(about) - page.takeErrors() - // A heading from the middle of the README: deep enough that folding - // hides it, and the kind of link a contents list makes. - const id = await page.evaluate(`(() => { - const ids = [...document.querySelectorAll('.readme [id]')].map((e) => e.id) - return ids[Math.floor(ids.length / 2)] ?? null - })()`) - if (!id) { - console.log(` - ${name.padEnd(7)} ${about}: no README heading, skipped`) - continue - } + await page.goto('about:blank') + await page.goto(BASE + about) + await arrive(about) + page.takeErrors() + // A heading from the middle of the README: deep enough that folding + // hides it, and the kind of link a contents list makes. + const id = await page.evaluate(`(() => { + const ids = [...document.querySelectorAll('.readme [id]')].map((e) => e.id) + return ids[Math.floor(ids.length / 2)] ?? null + })()`) + if (!id) { + console.log(` - ${name.padEnd(7)} ${about}: no README heading, skipped`) + continue + } - let s = await state() - expect('a first visit starts folded, at the top', s.open === false && s.y === 0, s) + let s = await state() + expect('a first visit starts folded, at the top', s.open === false && s.y === 0, s) - // Unfold and read down past where the folded page ends, so a page that - // came back folded could not land there. - const foldedBottom = await page.evaluate('document.documentElement.scrollHeight - innerHeight') + // Unfold and read down past where the folded page ends, so a page that + // came back folded could not land there. + const foldedBottom = await page.evaluate('document.documentElement.scrollHeight - innerHeight') - // The browser's own restore, on a page the fold leaves alone. Nuxt's - // start-up scroll to the top used to cancel it on every page but `/` - // (see app/router.options.ts), so this landed at 0 in both engines. - const halfway = Math.round(foldedBottom / 2) - await page.evaluate(`scrollTo({ top: ${halfway}, behavior: 'instant' })`) - await arrive(about) - const read = (await state()).y - await page.reload() - await arrive(about) - s = await state() - expect(`a reload of the folded page stays at y ${read}`, s.open === false && Math.abs(s.y - read) <= SLACK, s) - await page.evaluate(`scrollTo({ top: 0, behavior: 'instant' })`) - await arrive(about) + // The browser's own restore, on a page the fold leaves alone. Nuxt's + // start-up scroll to the top used to cancel it on every page but `/` + // (see app/router.options.ts), so this landed at 0 in both engines. + const halfway = Math.round(foldedBottom / 2) + await page.evaluate(`scrollTo({ top: ${halfway}, behavior: 'instant' })`) + await arrive(about) + const read = (await state()).y + await page.reload() + await arrive(about) + s = await state() + expect(`a reload of the folded page stays at y ${read}`, s.open === false && Math.abs(s.y - read) <= SLACK, s) + await page.evaluate(`scrollTo({ top: 0, behavior: 'instant' })`) + await arrive(about) - await click('details > summary') - await sleep(300) - const target = await page.evaluate(`Math.min(${foldedBottom} + 1500, document.documentElement.scrollHeight - innerHeight - 10)`) - await page.evaluate(`scrollTo({ top: ${target}, behavior: 'instant' })`) - // Where the reader is once the README's lazy images around them have - // loaded and moved things, not where they were sent. - await arrive(about) - const left = (await state()).y - const deep = left > foldedBottom + 100 - const near = (y) => !deep || Math.abs(y - left) <= SLACK + await click('details > summary') + await sleep(300) + const target = await page.evaluate(`Math.min(${foldedBottom} + 1500, document.documentElement.scrollHeight - innerHeight - 10)`) + await page.evaluate(`scrollTo({ top: ${target}, behavior: 'instant' })`) + // Where the reader is once the README's lazy images around them have + // loaded and moved things, not where they were sent. + await arrive(about) + const left = (await state()).y + const deep = left > foldedBottom + 100 + const near = (y) => !deep || Math.abs(y - left) <= SLACK - await click(`a[href="${blog}"]`) - await arrive(blog) - await page.evaluate('history.back()') - await arrive(about) - s = await state() - expect(`Back reopens the README at y ${left}`, s.open === true && near(s.y), s) + await click(`a[href="${blog}"]`) + await arrive(blog) + await page.evaluate('history.back()') + await arrive(about) + s = await state() + expect(`Back reopens the README at y ${left}`, s.open === true && near(s.y), s) - await page.evaluate('history.forward()') - await arrive(blog) - await page.evaluate('history.back()') - await arrive(about) - s = await state() - expect(`Forward and Back again, at y ${left}`, s.open === true && near(s.y), s) + await page.evaluate('history.forward()') + await arrive(blog) + await page.evaluate('history.back()') + await arrive(about) + s = await state() + expect(`Forward and Back again, at y ${left}`, s.open === true && near(s.y), s) - // What the page did on its way back, frame by frame — only read when - // the check fails, so a CI failure says where the pixels went. - const untrace = await page.preload(TRACE) - await page.reload() - await arrive(about) - await untrace() - s = await state() - const reloaded = s.open === true && near(s.y) - expect(`a reload reopens it at y ${left}`, reloaded, reloaded ? s : { ...s, trace: await page.evaluate('window.__trace ?? null') }) + // What the page did on its way back, frame by frame — only read when + // the check fails, so a CI failure says where the pixels went. + const untrace = await page.preload(TRACE) + await page.reload() + await arrive(about) + await untrace() + s = await state() + const reloaded = s.open === true && near(s.y) + expect(`a reload reopens it at y ${left}`, reloaded, reloaded ? s : { ...s, trace: await page.evaluate('window.__trace ?? null') }) - await click(`a[href="${blog}"]`) - await arrive(blog) - await click(`a[href="${about}"]`) - await arrive(about) - s = await state() - expect('a visit by link starts folded, at the top', s.open === false && s.y <= 1, s) + await click(`a[href="${blog}"]`) + await arrive(blog) + await click(`a[href="${about}"]`) + await arrive(about) + s = await state() + expect('a visit by link starts folded, at the top', s.open === false && s.y <= 1, s) - const hash = `#${encodeURIComponent(id)}` - await page.goto('about:blank') - // Slowed, in Chromium: the browser's own scroll to the fragment and - // vue-router's met in one frame, and the animation's first step could - // land on top of the jump — 4 or 18 px past the heading. At full speed - // that was 7 cold loads in 40 and CI red now and then; at 6× it was 14 - // in 20, so this fails reliably if it comes back. - await page.throttle(6) - await page.goto(BASE + about + hash) - await arrive(about) - await page.throttle(1) - s = await state(id) - expect(`a cold load of ${hash} opens it at the heading`, atHeading(s), s) + const hash = `#${encodeURIComponent(id)}` + await page.goto('about:blank') + // Slowed, in Chromium: the browser's own scroll to the fragment and + // vue-router's met in one frame, and the animation's first step could + // land on top of the jump — 4 or 18 px past the heading. At full speed + // that was 7 cold loads in 40 and CI red now and then; at 6× it was 14 + // in 20, so this fails reliably if it comes back. + await page.throttle(6) + await page.goto(BASE + about + hash) + await arrive(about) + await page.throttle(1) + s = await state(id) + expect(`a cold load of ${hash} opens it at the heading`, atHeading(s), s) - await click(`a[href="${blog}"]`) - await arrive(blog) - await page.evaluate(`document.querySelector('#__nuxt').__vue_app__.config.globalProperties.$router.push(${JSON.stringify(about + hash)})`) - await arrive(about) - s = await state(id) - expect(`a client navigation to ${hash} opens it at the heading`, atHeading(s), s) + await click(`a[href="${blog}"]`) + await arrive(blog) + await page.evaluate(`document.querySelector('#__nuxt').__vue_app__.config.globalProperties.$router.push(${JSON.stringify(about + hash)})`) + await arrive(about) + s = await state(id) + expect(`a client navigation to ${hash} opens it at the heading`, atHeading(s), s) + + await click('details > summary') + await sleep(300) + await click(`a[href="${blog}"]`) + await arrive(blog) + await page.evaluate('history.back()') + await arrive(about) + s = await state() + expect('folded by the visitor, it stays folded on Back', s.open === false, s) - await click('details > summary') - await sleep(300) - await click(`a[href="${blog}"]`) - await arrive(blog) - await page.evaluate('history.back()') + // A reader on a slow line opens the fold before the app has hydrated: + // it must stay open. Hydration once wrote the bound `false` over it. + const undo = await page.preload(`document.addEventListener('DOMContentLoaded', () => { document.querySelector('.readme')?.closest('details')?.setAttribute('open', '') })`) + await page.goto('about:blank') + await page.goto(BASE + about) await arrive(about) + await sleep(1500) + await undo() s = await state() - expect('folded by the visitor, it stays folded on Back', s.open === false, s) + expect('opened before hydration, it stays open', s.open === true, s) - // A reader on a slow line opens the fold before the app has hydrated: - // it must stay open. Hydration once wrote the bound `false` over it. - const undo = await page.preload(`document.addEventListener('DOMContentLoaded', () => { document.querySelector('.readme')?.closest('details')?.setAttribute('open', '') })`) - await page.goto('about:blank') - await page.goto(BASE + about) - await arrive(about) - await sleep(1500) - await undo() - s = await state() - expect('opened before hydration, it stays open', s.open === true, s) - - for (const { text } of page.takeErrors()) problems.push(`console: ${text}`) - console.log(` ${problems.length ? '✗' : '·'} ${name.padEnd(7)} ${about}${deep ? '' : ' (README too short to test the scroll position)'}`) - for (const p of problems) console.error(` ${p}`) - failures.push(...problems.map((p) => `${name} ${about}: ${p}`)) + for (const { text } of page.takeErrors()) problems.push(`console: ${text}`) + console.log(` ${problems.length ? '✗' : '·'} ${name.padEnd(7)} ${about}${deep ? '' : ' (README too short to test the scroll position)'}`) + for (const p of problems) console.error(` ${p}`) + failures.push(...problems.map((p) => `${name} ${about}: ${p}`)) + } + catch (err) { + // A page that stops answering — CDP and BiDi requests give up after + // 180 s (`scripts/lib/cdp.mjs`) — is a failure of this page, not the + // end of the run. Say where it was and whether its renderer was busy + // or idle, then carry on in a fresh tab. + const busy = await browser.busy().catch(() => null) + const where = `${/got no answer/.test(err.message) ? 'stalled' : 'failed'} after ${calls.join(' → ')}${busy === null ? '' : `; renderer ${busy > 0.5 ? 'busy' : 'idle'} (${busy.toFixed(2)} CPU s in 1 s)`}` + console.log(` ✗ ${name.padEnd(7)} /projects/${slug}/about`) + const message = err.message.split('\n')[0] + console.error(` ${message}\n ${where}`) + failures.push(`${name} /projects/${slug}/about: ${message} — ${where}`) + await Promise.race([page.close(), new Promise((r) => setTimeout(r, 5000))]).catch(() => {}) + page = traced(await browser.open(viewport)) + } } await page.close() } diff --git a/scripts/lib/browser.mjs b/scripts/lib/browser.mjs index 0705fca..15e556b 100644 --- a/scripts/lib/browser.mjs +++ b/scripts/lib/browser.mjs @@ -30,10 +30,26 @@ export async function startDriver(engine) { async function chromiumDriver() { const { cdp, chrome, stop } = await startBrowser() + const cpu = async () => { + const { processInfo } = await cdp.send('SystemInfo.getProcessInfo') + return new Map(processInfo.filter((p) => p.type === 'renderer').map((p) => [p.id, p.cpuTime])) + } return { name: 'Chromium', binary: chrome, stop, + /** + * CPU seconds the busiest renderer spent over the next second: about 1 + * when a page's main thread is spinning, about 0 when it is idle — say, + * waiting on timers. Asked of the browser, not the page, so it answers + * when the page itself does not. + */ + async busy() { + const before = await cpu() + await new Promise((r) => setTimeout(r, 1000)) + const after = await cpu() + return Math.max(0, ...[...after].map(([id, t]) => t - (before.get(id) ?? t))) + }, async open(viewport) { const { targetId } = await cdp.send('Target.createTarget', { url: 'about:blank' }) const { sessionId } = await cdp.send('Target.attachToTarget', { targetId, flatten: true }) @@ -112,6 +128,10 @@ async function firefoxDriver() { name: 'Firefox', binary: firefox, stop, + /** BiDi cannot see process CPU time. */ + async busy() { + return null + }, async open({ width, height, deviceScaleFactor = 1 }) { const { context } = await bidi.send('browsingContext.create', { type: 'tab' }) await bidi.send('browsingContext.setViewport', { From a1200eac6adf22145f66674b29550a6a24621af6 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Tue, 13 Oct 2026 23:35:27 -0500 Subject: [PATCH 11/13] chore(data): refresh the fallback snapshot Skidbladnir v1.4.4 (released 2026-10-13) and its README, crate download counts, push times. Every README image in it carries its size. data/projects.generated.json Co-Authored-By: Claude Opus 5.5 --- data/projects.generated.json | 170 +++++++++++++++++------------------ 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/data/projects.generated.json b/data/projects.generated.json index c8be978..bc53eb7 100644 --- a/data/projects.generated.json +++ b/data/projects.generated.json @@ -1,10 +1,10 @@ { - "generatedAt": "2026-10-13T05:41:38.511Z", + "generatedAt": "2026-10-14T04:35:19.462Z", "org": "basic-automation", "repos": { "artiqwest": { "repo": "artiqwest", - "fetchedAt": "2026-10-13T05:41:30.769Z", + "fetchedAt": "2026-10-14T04:35:07.689Z", "description": "basic http request over tor using arti.", "htmlUrl": "https://github.com/basic-automation/artiqwest", "homepage": null, @@ -62,7 +62,7 @@ }, "onyums": { "repo": "onyums", - "fetchedAt": "2026-10-13T05:41:32.064Z", + "fetchedAt": "2026-10-14T04:35:11.735Z", "description": "Onion-service server using axum.", "htmlUrl": "https://github.com/basic-automation/onyums", "homepage": null, @@ -121,13 +121,13 @@ }, "download": null, "crateVersion": "0.5.0", - "crateDownloads": 19762, + "crateDownloads": 19764, "crateUrl": "https://crates.io/crates/onyums", "docsUrl": "https://docs.rs/onyums" }, "nisaba": { "repo": "nisaba", - "fetchedAt": "2026-10-13T05:41:36.098Z", + "fetchedAt": "2026-10-14T04:35:16.314Z", "description": "Desktop app for managing one product catalog and syncing it across eBay, Squarespace, XMR Bazaar and Amazon. Rust + Tauri 2 + Nuxt 3, with sandboxed vendor plugins and optional P2P sync over Tor.", "htmlUrl": "https://github.com/basic-automation/nisaba", "homepage": null, @@ -163,7 +163,7 @@ }, "Skidbladnir": { "repo": "Skidbladnir", - "fetchedAt": "2026-10-13T05:41:36.862Z", + "fetchedAt": "2026-10-14T04:35:17.127Z", "description": "Free, open-source desktop app (Windows, Linux, macOS) that converts images to WebP, AVIF, JPEG XL and HEIC, with every still-image option of the official encoders (cwebp, avifenc, cjxl, heif-enc) as labelled controls.", "htmlUrl": "https://github.com/basic-automation/Skidbladnir", "homepage": "https://basicautomation.io/projects/skidbladnir/about", @@ -195,10 +195,17 @@ "license": "ISC", "defaultBranch": "master", "createdAt": "2019-03-10T04:37:55Z", - "pushedAt": "2026-10-12T20:20:43Z", + "pushedAt": "2026-10-13T15:34:12Z", "archived": false, - "readmeHtml": "

Skidbladnir

\n

\"CI\"\n\"Latest\n\"Platforms:

\n

Skidbladnir makes image files smaller by converting them to the modern formats:\nWebP, AVIF, JPEG XL and HEIC. It is for anyone who wants smaller images for\na website, an archive or a photo library without learning four command-line tools. Pick\nyour files, see the result beside the original, then convert one image, a batch or a\nwhole folder.

\n

Free and open source · Windows, macOS (experimental) and Linux · converts on your\ncomputer; the only network request is the in-app update check

\n

Download ·\nWhich file to pick ·\nProject page

\n

Each of these formats has an official encoder, and each one is a command-line tool with\ndozens of options that are easy to forget and easy to get wrong. Skidbladnir puts those\noptions in a window, each labelled with the flag it sets, instead of a single quality\nslider.

\n

For a still image, every option of cwebp, avifenc (with libaom), cjxl and\nheif-enc (with Kvazaar) is a control in the window, and Skidbladnir writes the same bytes\nas those tools built from the same library versions, checked in CI against the real tools,\nwith the exceptions listed under What "every option" means.

\n

\"The

\n

AVIF selected: avifenc's options, each labelled with its flag.

\n
\nWith WebP selected: the whole panel, and a preview

\"The

\n

WebP selected, with a file previewed: every cwebp option, each labelled with its flag.

\n

Features

\n

Input formats

\n
    \n
  • PNG, JPEG (including Photoshop's CMYK), TIFF, AVIF, JPEG XL and HEIC, as\nstill images
    • \n
  • WebP and GIF, still or animated
    • \n
  • PNM: binary PGM, PPM and PAM (8 or 16 bits, with or without alpha), and PFM\nfloating-point images
    • \n
  • Y4M video frames (the first frame): any chroma format, 8 to 16 bits, with or without\nalpha
    • \n
  • PGX, the JPEG 2000 test format (gray, 1 to 16 bits)
    • \n
  • OpenEXR (.exr): half or float RGB, RGBA or gray, any channel layer, every compression\nbut HTJ2K
    • \n
  • Raw YUV (.yuv): bare 8-bit I420 planes, at the size you give, as cwebp -s reads them
    • \n
  • SVG (and gzip-compressed .svgz), drawn at its own size, or straight at the size a resize asks for (only the crop, if there is one) so its edges\nstay sharp, with its transparency kept; text in the system's fonts, and only images\nembedded in the file (one it links to is never opened)
    • \n
  • Each file's format is recognised from its contents, not its extension (except raw YUV,\nwhich has no header to recognise). An AVIF's crop,\nrotation and mirror are applied, so a sideways-stored portrait converts upright, and\ntiled (grid) AVIFs, as some cameras write large captures, are decoded whole.
    • \n
\n

Output formats, each with its reference encoder's whole command line for a still image

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
FormatReference toolEncoder in the appByte-for-byte parity, tested
WebPcwebp 1.6.0libwebp 1.6.03,091 cases: every option, its no-option defaults, through PNG (every colour type, gamma), JPEG, TIFF (including premultiplied alpha, and straight alpha read as cwebp reads it on request), WebP and PNM files, raw YUV (-s), and -metadata
Animated WebPimg2webp, gif2webp and webpmux 1.6.0libwebp 1.6.0215 cases: animated WebP and GIF input, every Skidbladnir setting those tools can express, their -mixed, -min_size, -kmin/-kmax, -loop, -loop_compatibility and -d for every frame, img2webp's frame options (-lossy/-lossless, -q, -m, -exact/-noexact, -d) changed from any frame on, and metadata (gif2webp -metadata, webpmux -set)
AVIFavifenc 1.4.2libavif 1.4.2 + libaom 3.15.1231 cases: every option, PNG inputs, JPEG and Y4M input
JPEG XLcjxl 0.12.0libjxl 0.12.0551 cases: every option, PNG, PNM, PFM, PGX, EXR, still-GIF and JPEG XL inputs, JPEG recompression and decoding
HEICheif-enc -e kvazaar 1.23.5libheif 1.23.5 + Kvazaar 2.3.2182 cases: every option, PNG, JPEG, TIFF (RGB, straight alpha, and premultiplied alpha read as heif-enc reads it on request), WebP, HEIC and 8-bit 4:2:0 Y4M inputs
\n

That is 4,270 cases in all. CI runs every one of them on Linux x86-64, Linux ARM64,\nmacOS with Apple silicon and Windows x64, for every pull request and every push to\nmaster.

\n

The GPL edition writes HEIC with x265 instead, adding heif-enc's -L\nlossless, 4:4:4 and 4:2:2 chroma, 10-bit output, x265's main tuning controls and any\nother x265 parameter (-p x265:KEY=VALUE); its HEIC is checked by decoding, not byte for\nbyte against a reference.

\n
    \n
  • WebP: lossless and near-lossless, -exact, the libwebp presets and -z lossless\nlevels, targets by size or PSNR, every lossy tuning control (SNS, segments, filter\ntype, strength, sharpness and auto-filter, passes, quality range, pre-processing,\npartition limit, JPEG-like), alpha quality, method and filtering, -noalpha,\n-blend_alpha, and -metadata.
    • \n
  • AVIF: colour and alpha quality (or leave them to avifenc), speed, lossless, 8,\n10 or 12 bits and the 16-bit depth extension, 4:4:4, 4:2:2, 4:2:0 or 4:0:0, sharp YUV,\npremultiplied alpha, CICP and range, target size, progressive layers, grids, tiling,\nscaling, quantizers, libaom's own options, pixel aspect, clean aperture, rotation,\nmirroring, content light level, and ICC, Exif and XMP from the input, left out or from a\nfile.
    • \n
  • JPEG XL: distance or quality, alpha distance, effort (up to 11), lossless JPEG\nrecompression — a JPEG repacked typically about 20% smaller (libjxl's figure) and\nrebuildable bit for bit, and the default for JPEG input — and every one of cjxl's\nmodular, progressive, filter, colour-space, metadata and container options.
    • \n
  • HEIC, the format iPhones use: quality, Kvazaar's lossless coding, chroma\ndownsampling, the colour-profile presets and custom code points, thumbnails, alpha and\npremultiplied alpha, rotation and mirroring, tiles, 360° projection, a description,\ncompatible brands and the compact mini format.
    • \n
  • Metadata: a photo's ICC colour profile, Exif and XMP are kept as each reference\ntool keeps them: in WebP when you ask, as cwebp -metadata does (off by default, as in\ncwebp); in AVIF, JPEG XL and HEIC by default, as avifenc, cjxl and heif-enc do,\nwith each kind switchable off.
    • \n
  • Crop and resize (with -resize_mode), shared by every format.
    • \n
\n

Workflow

\n
    \n
  • Preview the result beside the original, in any format, before anything is written to\ndisk. AVIF and JPEG XL previews are decoded in Rust, so they show on every platform.
    • \n
  • Animated WebP or GIF in, animated WebP out: every frame, its timing and its loop count\nkept (or a loop count of your own), every WebP setting applied to each frame, the\nanimation encoder's keyframe spacing, mixed lossy/lossless and minimum-size options,\nand a preview of the animation first.\nA GIF converts as gif2webp converts it, across disposal methods, transparency, timing\nand loop counts, keeping its ICC profile and XMP when you ask, as\ngif2webp -metadata does.
    • \n
  • Drop files onto the window (any it cannot read are set aside), or convert a whole\nfolder, optionally with its subfolders, recreating their structure in the destination.
    • \n
  • Batch conversion with progress per file, and a Cancel button that never leaves a\nhalf-written image behind.
    • \n
  • The original and converted sizes, and the dimensions produced, reported after each\nconversion; and what an existing WebP already is: size, lossy or lossless, alpha,\nanimated.
    • \n
  • Each format keeps its own settings while you try another. Settings and the destination\nare remembered between launches, and you can save named presets of your own.
    • \n
  • Every control shows the flag it sets, with the expert ones in folding sections. The\nwhole window works from the keyboard, and every control, text and number fields\nincluded, has a name and a role a screen reader can announce. CI checks both on every\nchange: it tabs through each format's panel and checks every control is reached, and\nreads the window the way a screen reader does, over AT-SPI on Linux and UI Automation\non Windows (and with axe in the webview window). It also drops a file onto the window\nas a file manager does, through X11 on Linux and by dragging it out of Explorer on\nWindows, and converts it.
    • \n
  • Your source image is never overwritten, and every write goes through a temporary file,\nso a failed conversion cannot damage a file that was already there. Before a run, the\nwindow warns when two queued files would be written to the same name (photo.png and\nphoto.jpg both become photo.webp) and when outputs are already in the destination.\nThose are replaced, as cwebp -o would replace them, unless you turn off Replace\nexisting files in the app settings, which keeps them and skips those inputs.
    • \n
  • About and licences, in the app settings, shows the version, the edition and its\nlicence, and the licence texts and third-party notices the app carries.
    • \n
\n

Install

\n

Download the file for your computer from the\nlatest release\n(<version> is the release's version number):

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Your computerFile to download
Windows 10 or 11, 64-bitSkidbladnir_<version>_x64-setup.exe
Mac with Apple silicon (M1 or later), experimentalSkidbladnir_<version>_aarch64.dmg
Mac with an Intel processor, experimentalSkidbladnir_<version>_x64.dmg
Ubuntu 22.04 or later, Debian 12 or later, Linux Mint 21 or later, and distributions based on themSkidbladnir_<version>_amd64.deb
Fedora, openSUSE and other RPM-based distributions with glibc 2.35 or laterSkidbladnir_<version>_x86_64.rpm
Any other 64-bit Linux with glibc 2.35 or laterSkidbladnir_<version>_amd64.AppImage
Linux on 64-bit ARM (a Raspberry Pi 4 or 5 on a 64-bit system, ARM laptops and servers), glibc 2.35 or laterSkidbladnir_<version>_arm64.deb, Skidbladnir_<version>_aarch64.rpm or Skidbladnir_<version>_aarch64.AppImage, as above
\n

Not sure which edition? Download the standard edition: the files above, named\nSkidbladnir_… with an underscore. Choose the GPL edition, the same files named\nSkidbladnir-GPL_…, only if you write HEIC and want the x265 encoder: heif-enc -L\nlossless, 4:4:4 and 4:2:2 chroma, 10-bit, x265's preset and tune, its main tuning\ncontrols, and any other x265 parameter. Both are free. The licence matters only if you\nredistribute the app; Editions has the details. On ARM Linux the GPL\nedition's x265 runs without its assembly (it cannot be linked into the shared libheif\nthere), so its HEIC encodes are slower; the files are the same.

\n

The release page lists other files too; they are not installers. The .app.tar.gz files\nare for the in-app updater on a Mac. Skidbladnir-<version>-source.tar.gz is the complete\nsource with every submodule; GitHub's own "Source code" archives leave the submodules out.

\n

There are no builds for 32-bit systems or Windows on ARM. Windows on ARM can run x64\nprograms through emulation, but nobody has tried Skidbladnir there.\nStatus says which builds have been run, and how.

\n

Opening it the first time

\n

The installers are not code-signed yet, so Windows and macOS warn before they open an app\nfrom a developer they cannot identify. This is what to expect, and how to get past it.

\n

Windows. Your browser may warn that the file is not commonly downloaded. Keep it\nanyway. In Edge, open the ... (More actions) menu beside the download, choose\nKeep, then Show more > Keep anyway. In Chrome, choose Keep or Download\nsuspicious file. When you run the installer, SmartScreen shows Windows protected your\nPC: click More info, then Run anyway. Skidbladnir installs for your account\nonly, in %LOCALAPPDATA%\\Skidbladnir, without administrator rights. If Microsoft's\nWebView2 runtime is missing (Windows 10 and 11 normally have it), the installer downloads\nit. If Smart App Control is turned on (Windows Security > App & browser control),\nWindows blocks unsigned apps and offers no exception, so Skidbladnir cannot be installed\non that PC until it is signed.

\n

macOS (experimental). The macOS builds are new: CI builds them, runs the test suite\non macOS and converts to every format through the window there, but nobody has opened the\napp on a real Mac yet, so please\nreport problems. Open the .dmg,\ndrag Skidbladnir into Applications, and open it from there.

\n
    \n

  • On macOS 15 (Sequoia) or later, the first time you open it, macOS says it cannot\nverify the app. Click Done, open System Settings > Privacy & Security, scroll\ndown, click Open Anyway beside the message about Skidbladnir, and confirm.

    \n
    • \n

  • On macOS 14 or earlier, right-click (or Control-click) Skidbladnir in Applications,\nchoose Open, then choose Open again.

    \n
    • \n

  • If macOS says that Skidbladnir is damaged and can't be opened, there is no Open Anyway\nbutton. Remove the quarantine flag your browser set, in Terminal, then open it again:

    \n
    xattr -dr com.apple.quarantine /Applications/Skidbladnir.app
    • \n
\n

Linux. Both packages need glibc 2.35 or later: Ubuntu 22.04, Debian 12 or newer.

\n
    \n
  • The .deb (Ubuntu, Debian, Linux Mint): in the folder you downloaded it to, run\nsudo apt install ./Skidbladnir_<version>_amd64.deb. apt installs what it needs\n(xkbcommon and the Vulkan loader for the window, and WebKitGTK and GTK for the webview\nwindow it falls back to) and adds Skidbladnir to your applications menu.
    • \n
  • The AppImage (any distribution): run chmod +x Skidbladnir_<version>_amd64.AppImage,\nthen ./Skidbladnir_<version>_amd64.AppImage. It adds no menu entry. It needs FUSE: a\nfusermount3 or fusermount program (the fuse3 or fuse package) and /dev/fuse;\nlibfuse2 is not needed. Where FUSE is not available, run it with\n--appimage-extract-and-run.
    • \n
\n

Checking a download

\n

The release page lists a SHA-256 checksum beside each file. Work out the checksum of the\nfile you downloaded and compare the two:

\n
    \n
  • Linux: sha256sum Skidbladnir_<version>_amd64.deb
    • \n
  • macOS: shasum -a 256 Skidbladnir_<version>_aarch64.dmg
    • \n
  • Windows, in PowerShell: Get-FileHash .\\Skidbladnir_<version>_x64-setup.exe (it uses\nSHA-256 unless told otherwise)
    • \n
\n

If they differ, the file is not the one the release published: delete it and download it\nagain.

\n

Updates

\n

Once installed, Skidbladnir checks for a newer release each time it starts and offers it\nin a banner; nothing is downloaded until you click Install and restart, and every\nupdate is verified against the project's signing key before it is installed. A .deb\ninstall updates with a .deb and an .rpm install with an .rpm (each asks for your\npassword to do it); an AppImage, the\nWindows installer and the macOS app update themselves in place. Each edition updates only\nto the same edition; to switch, install the other one over it. Releases before 0.8.0\nhave no updater, so moving off them is a manual download, once.

\n

Privacy

\n

Skidbladnir has no telemetry and no analytics, and your images never leave your\ncomputer. The app makes one network request of its own: when it starts, it fetches a\nsmall update manifest (latest.json, or latest-gpl.json for the GPL edition) from\nraw.githubusercontent.com. Like any web request, that shows GitHub your IP address;\nnothing about you or your files is sent. An update is downloaded from GitHub, and\ninstalled, only when you click Install and restart.

\n

Editions

\n

The two editions are the same app except for how they write HEIC, and so for their\nlicence:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Standard (Skidbladnir_*)GPL (Skidbladnir-GPL_*)
HEIC encoderKvazaar (BSD-3-Clause)x265 (GPL-2.0-or-later)
HEIC controlsevery heif-enc option Kvazaar honoursthose, plus -L lossless, 4:4:4 and 4:2:2 chroma, 10-bit, and x265's preset, tune, TU depth, AQ, psy-rd/psy-rdoq, deblock and SAO as controls, with any other -p x265: parameter by name
HEIC checkedbyte for byte against heif-enc -e kvazaarby decoding
Licence of the buildISC; ships libheif/libde265 as a separate, replaceable LGPL-3.0 libraryGPL-3.0-or-later, as a whole
\n

The GPL edition exists for people who want x265, the HEVC encoder a default heif-enc\nbuild uses, and x265 is GPL: an app that ships it is distributed under the GPL.\nSkidbladnir's own source is ISC either way. Everything but HEIC output — WebP, AVIF,\nJPEG XL, and HEIC input — is identical in both. In both, libheif ships as a separate\nshared library beside the app, with its codec plugins switched off, so the standard\nedition can never pick up x265 from your system.

\n

Each release attaches Skidbladnir-<version>-source.tar.gz, the complete source with\nevery submodule, which is what the GPL edition's licence requires to be offered with it.\nThe Rust crates and npm packages it compiles in are pinned by Cargo.lock and\nfrontend/package-lock.json in that archive, and published unmodified on crates.io and\nnpmjs.com.

\n

What "every option" means

\n

The claim is tested, not assumed: for each format, a test runs the reference tool and\nSkidbladnir on the same input with the same options, across the whole option surface and\nacross the inputs the tool reads differently (PNG colour types and chunks, JPEG chroma\nsamplings, CMYK, metadata), and requires identical bytes. CI builds each reference from the\nsame source as the library the app links (scripts/build-reference-tools.sh). Precisely:

\n
    \n
  • Still images, and animated WebP out of an animated WebP or a GIF, as img2webp and\ngif2webp write it, including their keyframe (-kmin/-kmax), -mixed, -min_size,\n-loop and -loop_compatibility options and img2webp -d for every frame. A different\nquality or duration for each frame is not offered yet. Neither are other options\nabout more than one image: animated AVIF, HEIC or JPEG XL, image sequences,\navifenc --layered and grids assembled from several files, heif-enc with several\ninputs or -T tiled input, and cjxl from an animated GIF or APNG. They are queued in\nROADMAP.md, Phase 8.
    • \n
  • The input formats Skidbladnir reads: PNG, JPEG, TIFF, WebP, AVIF, JPEG XL, HEIC,\nGIF, PNM/PAM, PFM, PGX, Y4M, SVG, EXR and raw YUV. Raw YUV is read as cwebp -s reads it:\ninto WebP as its planes are, byte for byte, and converted to RGB by libwebp for the other\nformats, whose tools do not read it. EXR is read as cjxl reads it — linear light, its\npremultiplied alpha, chromaticities and white luminance, and its other channels kept as\nnamed extra channels — and matched byte for byte. Only cjxl reads EXR, so for the other formats it is given the\nsRGB curve, clipped at white. A JPEG XL converted to JPEG XL is decoded by libjxl as\ncjxl decodes it — its own colour, extra channels, Exif (orientation reset, as the pixels\nare turned upright), XMP, IPTC and its other boxes — and matched byte for byte, except\nthat a JPEG XL of several frames or layers is converted from its pixels.\n(heif-enc --raw cannot write a HEIC in this build, so there is nothing of it to match.) None of them reads SVG, so an SVG has no reference to\nmatch: it is drawn by resvg. cjxl reads only 8- and 16-bit PGX at the right brightness\n(a 12-bit file comes out at a sixteenth of it), so those two depths are matched byte\nfor byte and the others are read as meant. A Y4M is read by avifenc's own reader and encoded to AVIF as\nits planes are, and an 8-bit 4:2:0 one goes to HEIC as heif-enc reads it. heif-enc\nreads every other Y4M as if it were 8-bit 4:2:0 too, garbling it, so those are\nconverted to RGB (BT.601) for HEIC instead, as they are for WebP and JPEG XL, which no\nreference tool reads Y4M for. PNM is read the way cwebp reads it for WebP and the way\ncjxl reads it for JPEG XL (at its own bit depth); PFM is read as cjxl reads it, and\nonly cjxl reads PFM, so PFM to another format has no reference to match. JPEG is read\nthrough libjpeg-turbo 3.2.0, the way each tool reads it — including avifenc's and\nheif-enc's copying of a JPEG's own YCbCr planes — and PNG by libpng 1.6's rules,\nincluding cwebp's gamma correction. For WebP, 16-bit files are reduced to 8 bits the\nway cwebp's libpng or libtiff reduces them, a WebP source takes cwebp's route\nstraight to YUV, and a TIFF with premultiplied (associated) alpha is un-multiplied as\ncwebp un-multiplies it. A tool linked against a different libjpeg or libpng can\ndecode the same file differently.
    • \n
  • The same encoders, built the same way: libwebp 1.6.0, libavif 1.4.2 with libaom\n3.15.1, libjxl 0.12.0, libheif 1.23.5 with Kvazaar 2.3.2. Another version of a tool\nwrites other bytes, as it would against itself. The app's libavif and the reference\navifenc are both built without libyuv. A packaged avifenc (Homebrew, most Linux\ndistributions) usually links libyuv, which converts RGB to YUV differently, so it\nwrites other bytes even at libavif 1.4.2.
    • \n
  • Tested on Linux (x86-64 and ARM64), macOS on Apple silicon and Windows x64. CI\nbuilds the reference tools and runs the parity tests on each, so the encoders' x86 and\nARM code paths, and their MSVC builds, are each held to the tools. One exception in\nwhat is compared: for an EXR whose data window does not cover its display window, cjxl\nencodes whatever memory it was given for the uncovered pixels (libjxl 0.12.0 never writes\nthem): zero on Linux and macOS by chance, other bytes on Windows. Skidbladnir makes them zero, and those\ncases are held to cjxl reading the same image with the zeros written out. The Intel Mac build\nlinks the same sources, but no reference tool runs there, so identical bytes there are\nexpected, not tested.
    • \n
  • AVIF with libaom, avifenc's default codec: -c rav1e and -c svt are not built\nin. JPEG gain-map conversion needs avifenc built with libxml2, which it is not by\ndefault, and is not offered.
    • \n
  • HEIC with Kvazaar in the standard edition, the BSD-licensed HEVC encoder; the\ndefault heif-enc build uses x265, which is GPL, and is the GPL edition's\nencoder. heif-enc -L fails with Kvazaar, so lossless is Kvazaar's own\n(-p lossless=true), and 16-bit PNGs, which heif-enc hands Kvazaar at 10 bits and\nKvazaar refuses, are encoded from their high bytes instead. The GPL edition's x265\noutput is checked by decoding it, not byte for byte.
    • \n
  • WebP's starting settings. Until you change them, the WebP controls start from\nSkidbladnir's own long-standing defaults (-af -alpha_filter best -pass 6 -mt), not\nfrom what cwebp does with no flags. To start where cwebp starts, click Start from\ncwebp's defaults in the WebP panel (tested against cwebp given no options at all),\nthen set the flags you use.
    • \n
  • Options that change how a tool runs but not the file — verbosity, timing, benchmarks,\nprinting statistics — have no control, and neither do cwebp's dump and map outputs.
    • \n
\n

Where a tool refuses an input — cwebp a CMYK JPEG, heif-enc an RGB-coded one —\nSkidbladnir converts it anyway. Two tools misread a TIFF's transparency, and by default\nSkidbladnir reads it correctly instead, keeping the colours the file means:

\n
    \n
  • Straight (unassociated) alpha, to WebP: cwebp reads such a TIFF premultiplied,\nthrough libtiff, and darkens its semi-transparent pixels.
    • \n
  • Premultiplied (associated) alpha, to HEIC: heif-enc takes that alpha as straight,\nso its HEIC is darker where the image is semi-transparent. (Skidbladnir un-multiplies\nit, as cwebp does.)
    • \n
\n

To get the tools' results instead, turn on TIFF input → Read transparency as the\nofficial tool does: both are then reproduced byte for byte (tested against cwebp and\nheif-enc). avifenc and cjxl do not read TIFF, so AVIF and JPEG XL are unaffected.

\n

Besides the byte comparisons, the output is decoded with each format's own tools. AVIF is\ndecoded with libavif's avifdec, and the tests measure how close the pixels come back and\ncheck that transparency survives. JPEG XL is decoded with libjxl's djxl, which must\ndecode it at the right size and alpha, a lossless file exactly, and rebuild a recompressed\nJPEG byte for byte. HEIC is decoded with libheif's heif-dec and heif-info, from a\nseparate build; that is how the GPL edition's x265 output is checked. Metadata is checked\nagainst cwebp itself and with avifdec, djxl and heif-info. AVIF input is checked\nagainst avifdec across 17 colour encodings (bit depths, chroma subsampling, range,\ncolour matrices, alpha), and JPEG XL input against djxl.

\n

How it compares

\n

As each project describes itself on its own page, checked on 1 October 2026. "Not\nstated" means its page does not say, not that it cannot.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Runs asBatchWritesEncoder optionsByte parity with the reference tools
Skidbladnirdesktop app: Windows, macOS, Linux (x86-64 and ARM64)yes, files or whole foldersWebP, AVIF, JPEG XL, HEICevery option of cwebp, avifenc, cjxl and heif-enc for a still imageyes, tested: 4,270 cases (above)
Squooshweb app in the browser (installable), nothing uploadednot statedMozJPEG, WebP, AVIF, JPEG XL, OxiPNG, QOI, WebP 2 (its codec list); no HEICnot statednot stated
XnConvertdesktop app: Windows, macOS, Linux; free for private or educational use, paid in a companyyesabout 70 formats, including WebP, AVIF, JPEG XL and HEICnot statednot stated
Converseendesktop app on ImageMagick: Windows, Linux, macOS, FreeBSD; GPL-3.0yesover 100 formats, including WebP, AVIF and HEIC; JPEG XL not mentionednot statednot stated
The reference command-line toolscommand lineby scriptingone format eachall of them, by definitionthey are the reference
\n

Build from source

\n

Requires a stable Rust toolchain, Node.js and npm, nasm — libaom, libjpeg-turbo and\nthe AV1 decoder assemble their SIMD code with it, and x265 in the GPL edition (Arch:\nnasm; Debian/Ubuntu: nasm; macOS: brew install nasm; Windows: choco install nasm)\n— and CMake with a C++ compiler, which build libjxl, libavif with libaom, and\nlibjpeg-turbo from source so they are linked statically. Clone with\n--recurse-submodules: those sources are pinned under third_party/. HEIC needs libheif,\nbuilt with scripts/build-libheif.sh from the same pinned sources into build/libheif;\nwithout that build, the system libheif is linked instead. On Linux you also need\nthe development headers of webkit2gtk-4.1, and of xkbcommon (with xkbcommon-x11), xcb,\nWayland, fontconfig and FreeType for the gpui window.

\n
npm --prefix frontend install        # also installs the Tauri CLI\ncd src-tauri && ../frontend/node_modules/.bin/tauri build --no-bundle

Run it with ../frontend/node_modules/.bin/tauri dev from src-tauri/, which starts the\nNuxt dev server and the window together. Release builds must go through tauri build\nrather than cargo build --release: the custom-protocol feature that embeds the\nfrontend is only enabled by the former.

\n

Without CMake, the build links the system libjxl, libavif and libjpeg through\npkg-config instead, and says so. Set SKIDBLADNIR_LIBJXL, SKIDBLADNIR_LIBAVIF or\nSKIDBLADNIR_LIBJPEG to vendored to insist on the static build, or to system to insist\non the system library.

\n

The GPL edition is the gpl feature (or full, every optional feature), with a libheif\nbuilt around x265 into build/libheif-gpl. The build refuses to link either edition\nagainst the other's libheif.

\n
scripts/build-libheif.sh --edition gpl\ncd src-tauri && ../frontend/node_modules/.bin/tauri build --features gpl --config tauri.gpl.conf.json

tauri.gpl.conf.json (and on Windows tauri.gpl.windows.conf.json too) bundles x265's\nlibheif, the GPL edition's notices and the GPL text, and points the updater at the GPL\nedition's releases.

\n

The encoders are the libraries themselves, linked into the binary — there is no command-line\ntool to download and no subprocess.

\n

Checking a build

\n
cargo test --workspace     # includes the encoder-parity tests\nscripts/smoke-test.sh      # drives the webview window: renders, IPC, a real conversion\nscripts/a11y-audit.sh      # axe-core against the live webview window\nscripts/gpui-smoke.sh      # opens the gpui window and converts to every format

The parity tests compare each encoder with its reference tool and need them built from\nthe same sources as the libraries the app links, reading JPEG through the same\nlibjpeg-turbo: scripts/build-reference-tools.sh builds cwebp, img2webp, gif2webp,\navifenc, cjxl and heif-enc and prints the SKIDBLADNIR_REFERENCE_* variables that\npoint the tests at them. The decoding checks use libavif's avifdec and avifenc,\nlibjxl's djxl and libheif's heif-dec and heif-info from PATH. Missing tools make\nthe tests say so and skip rather than pass quietly; set SKIDBLADNIR_REQUIRE_PARITY=1 to turn a skip\ninto a failure.

\n

The two scripts need tauri-driver (cargo install tauri-driver) and WebKitWebDriver\n(webkit2gtk-driver on Debian/Ubuntu, webkitgtk-6.0 on Arch). They skip, loudly, when\nthose are missing.

\n

Status

\n

Skidbladnir is a Rust + Tauri 2 app for Windows, Linux and macOS, and it builds and\npasses its tests on all three in CI. Since 1.3.0 its window is drawn natively with\ngpui (crates/skidbladnir-gpui): the same window, control for\ncontrol, in about a third of the memory and starting about 30% sooner. The Nuxt +\nTailwind webview window it replaces is still built in. It opens when the app is started\nwith SKIDBLADNIR_UI=webview, or by itself when the gpui window cannot start (on a\nmachine with no GPU driver it can use, say). Releases are\nstable from 1.0.0; the ones before it were prereleases. The work queue lives in\nROADMAP.md.

\n

Skidbladnir used to be an Electron app for Windows. That app has been retired: its last\nbinary release is v0.4.3\n(2019), and its last source is the\nelectron-final tag.

\n

Known gaps:

\n
    \n
  • macOS is experimental. The .dmgs (Apple silicon and Intel) are built by CI, and\nthe test suite runs on macOS, where CI also opens the window and converts to every\nformat with it, but nobody has yet opened the app on a real Mac. The\nstandard edition's Windows installer is installed and exercised by CI on every change,\nand the Linux app (both editions) is launched and driven by CI on every change, though\nnot from its .deb or AppImage. Each release's .deb is installed by CI on Ubuntu 22.04\nand Debian 12, and its .rpm on the latest Fedora, on x86-64 and ARM64 alike, and\nchecked to find every library it needs; the ARM64 app itself has not been launched. The\npublished 0.5.0 .deb and AppImage were run through that release's smoke test by hand.
    • \n
  • Nobody has used Skidbladnir with a screen reader yet (Orca, Narrator, NVDA or\nVoiceOver). What one is given is checked automatically on Linux and Windows, not on\nmacOS. Before 1.4.1 the window's text and number fields (sizes, crops, exact values)\nwere missing from it entirely.
    • \n
  • The app is not code-signed on any platform;\nOpening it the first time shows how to get past each\nsystem's warning.
    • \n
  • An AVIF, JPEG XL or HEIC encode cannot be cancelled mid-file: none of those encoders\nreports progress, so Cancel takes effect when the current file finishes (which is then\ndiscarded, not written).
    • \n
  • An animated AVIF, JPEG XL or HEIC is read as one still image: its primary image or\nfirst frame.
    • \n
  • Animated AVIF, JPEG XL and HEIC are not written (see below), so an animation's metadata\nis kept in animated WebP only: a GIF's ICC profile and XMP as gif2webp -metadata keeps\nthem, an animated WebP's ICC profile, Exif and XMP as webpmux -set would set them.
    • \n
  • An animation (animated WebP or GIF) converts to WebP only. AVIF, JPEG XL and HEIC\noutput refuse it by name rather than keeping just the first frame, because those\nencoders write still images here. A target size or PSNR applies to each frame of an\nanimation, not to the whole file.
    • \n
  • JPEG XL files open in Safari, and in Firefox and Chrome as each enables it by default;\nuntil then, most web pages cannot show them. The long-promised JPEG 2000 was dropped\nas a goal; it has no momentum outside medical and archival imaging.
    • \n
  • The standard edition writes HEIC 8-bit 4:2:0 only (Kvazaar), so its lossless coding is\nlossless after the conversion to 4:2:0, as with heif-enc; -L lossless, 4:4:4 and\n10-bit take the GPL edition. Neither writes 12-bit HEIC.
    • \n
  • HEVC patents. HEIC is HEVC (H.265) inside HEIF, and HEVC is covered by patents\nlicensed through patent pools such as Access Advance and Via LA. Skidbladnir includes\nopen-source HEVC software: Kvazaar (x265 in the GPL edition) to write HEIC, and libde265\nto read it. The project holds no HEVC patent licence and passes none on. Whether you\nneed one depends on where you are and what you do with the files; this is not legal\nadvice.
    • \n
\n

Acknowledgements

\n

Skidbladnir is a window around other people's encoders, and is only as good as they are:\nlibwebp (the WebM project, Google),\nlibavif and\nlibaom (the Alliance for Open Media),\nlibjxl (the JPEG XL authors),\nlibheif and\nlibde265 (struktur AG),\nKvazaar (Ultra Video Group, Tampere University),\nx265 (MulticoreWare) and\nlibjpeg-turbo (which includes the work of the Independent\nJPEG Group). The decoders are rav1d through\navif-decode, and\njxl-oxide. The window is\nTauri, Nuxt, Nuxt UI and\nTailwind CSS, set in Fira Code,\nwith icons from the collections THIRD-PARTY-NOTICES.md credits.

\n

Skidbladnir is an independent project. It is not affiliated with, sponsored or endorsed\nby Google, the Alliance for Open Media, the JPEG committee, Apple, Microsoft, or any of\nthe projects above. WebP, AVIF, JPEG XL, HEIC and the other names here are used only to\nsay which formats and tools it works with; they belong to their owners.

\n

License

\n

Skidbladnir's own code is ISC (LICENSE), as declared in the workspace\nCargo.toml, apart from code adapted from the encoders, which keeps its licence:\ncrates/skidbladnir-encode/native/avif_shim.c carries portions of libavif's avifenc\nunder BSD-2-Clause.

\n

The standard edition's own code is ISC; the libraries it ships keep their licences,\nincluding LGPL-3.0 for libheif and libde265, which ship as a separate shared library you\nmay replace with your own build. The GPL edition's builds include x265, so each of them\nis distributed as a whole under the GNU GPL, version 3 or later\n(LICENSES/GPL-3.0.txt); see Editions.

\n

The native libraries each edition ships, the JavaScript packages in its webview window and\nthe Rust crates compiled into it, with their licences, are listed in\nTHIRD-PARTY-NOTICES.md and\nTHIRD-PARTY-NOTICES-GPL.md, and every installer carries its\nown edition's copy with its licence texts.

\n", + "readmeHtml": "

Skidbladnir

\n

\"CI\"\n\"Latest\n\"Platforms:

\n

Skidbladnir makes image files smaller by converting them to the modern formats:\nWebP, AVIF, JPEG XL and HEIC. It is for anyone who wants smaller images for\na website, an archive or a photo library without learning four command-line tools. Pick\nyour files, see the result beside the original, then convert one image, a batch or a\nwhole folder.

\n

Free and open source · Windows, macOS (experimental) and Linux · converts on your\ncomputer; the only network request is the in-app update check

\n

Download ·\nWhich file to pick ·\nProject page

\n

Each of these formats has an official encoder, and each one is a command-line tool with\ndozens of options that are easy to forget and easy to get wrong. Skidbladnir puts those\noptions in a window, each labelled with the flag it sets, instead of a single quality\nslider.

\n

For a still image, every option of cwebp, avifenc (with libaom), cjxl and\nheif-enc (with Kvazaar) is a control in the window, and Skidbladnir writes the same bytes\nas those tools built from the same library versions, checked in CI against the real tools,\nwith the exceptions listed under What "every option" means.

\n

\"The

\n

AVIF selected: avifenc's options, each labelled with its flag.

\n
\nWith WebP selected: the whole panel, and a preview

\"The

\n

WebP selected, with a file previewed: every cwebp option, each labelled with its flag.

\n

Features

\n

Input formats

\n
    \n
  • PNG, JPEG (including Photoshop's CMYK), TIFF, AVIF, JPEG XL and HEIC, as\nstill images
    • \n
  • WebP and GIF, still or animated
    • \n
  • PNM: binary PGM, PPM and PAM (8 or 16 bits, with or without alpha), and PFM\nfloating-point images
    • \n
  • Y4M video frames (the first frame): any chroma format, 8 to 16 bits, with or without\nalpha
    • \n
  • PGX, the JPEG 2000 test format (gray, 1 to 16 bits)
    • \n
  • OpenEXR (.exr): half or float RGB, RGBA or gray, any channel layer, every compression\nbut HTJ2K
    • \n
  • Raw YUV (.yuv): bare 8-bit I420 planes, at the size you give, as cwebp -s reads them
    • \n
  • SVG (and gzip-compressed .svgz), drawn at its own size, or straight at the size a resize asks for (only the crop, if there is one) so its edges\nstay sharp, with its transparency kept; text in the system's fonts, and only images\nembedded in the file (one it links to is never opened)
    • \n
  • Each file's format is recognised from its contents, not its extension (except raw YUV,\nwhich has no header to recognise). An AVIF's crop,\nrotation and mirror are applied, so a sideways-stored portrait converts upright, and\ntiled (grid) AVIFs, as some cameras write large captures, are decoded whole.
    • \n
\n

Output formats, each with its reference encoder's whole command line for a still image

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
FormatReference toolEncoder in the appByte-for-byte parity, tested
WebPcwebp 1.6.0libwebp 1.6.03,091 cases: every option, its no-option defaults, through PNG (every colour type, gamma), JPEG, TIFF (including premultiplied alpha, and straight alpha read as cwebp reads it on request), WebP and PNM files, raw YUV (-s), and -metadata
Animated WebPimg2webp, gif2webp and webpmux 1.6.0libwebp 1.6.0215 cases: animated WebP and GIF input, every Skidbladnir setting those tools can express, their -mixed, -min_size, -kmin/-kmax, -loop, -loop_compatibility and -d for every frame, img2webp's frame options (-lossy/-lossless, -q, -m, -exact/-noexact, -d) changed from any frame on, and metadata (gif2webp -metadata, webpmux -set)
AVIFavifenc 1.4.2libavif 1.4.2 + libaom 3.15.1231 cases: every option, PNG inputs, JPEG and Y4M input
JPEG XLcjxl 0.12.0libjxl 0.12.0551 cases: every option, PNG, PNM, PFM, PGX, EXR, still-GIF and JPEG XL inputs, JPEG recompression and decoding
HEICheif-enc -e kvazaar 1.23.5libheif 1.23.5 + Kvazaar 2.3.2182 cases: every option, PNG, JPEG, TIFF (RGB, straight alpha, and premultiplied alpha read as heif-enc reads it on request), WebP, HEIC and 8-bit 4:2:0 Y4M inputs
\n

That is 4,270 cases in all. CI runs every one of them on Linux x86-64, Linux ARM64,\nmacOS with Apple silicon and Windows x64, for every pull request and every push to\nmaster.

\n

The GPL edition writes HEIC with x265 instead, adding heif-enc's -L\nlossless, 4:4:4 and 4:2:2 chroma, 10-bit output, x265's main tuning controls and any\nother x265 parameter (-p x265:KEY=VALUE); its HEIC is checked by decoding, not byte for\nbyte against a reference.

\n
    \n
  • WebP: lossless and near-lossless, -exact, the libwebp presets and -z lossless\nlevels, targets by size or PSNR, every lossy tuning control (SNS, segments, filter\ntype, strength, sharpness and auto-filter, passes, quality range, pre-processing,\npartition limit, JPEG-like), alpha quality, method and filtering, -noalpha,\n-blend_alpha, and -metadata.
    • \n
  • AVIF: colour and alpha quality (or leave them to avifenc), speed, lossless, 8,\n10 or 12 bits and the 16-bit depth extension, 4:4:4, 4:2:2, 4:2:0 or 4:0:0, sharp YUV,\npremultiplied alpha, CICP and range, target size, progressive layers, grids, tiling,\nscaling, quantizers, libaom's own options, pixel aspect, clean aperture, rotation,\nmirroring, content light level, and ICC, Exif and XMP from the input, left out or from a\nfile.
    • \n
  • JPEG XL: distance or quality, alpha distance, effort (up to 11), lossless JPEG\nrecompression — a JPEG repacked typically about 20% smaller (libjxl's figure) and\nrebuildable bit for bit, and the default for JPEG input — and every one of cjxl's\nmodular, progressive, filter, colour-space, metadata and container options.
    • \n
  • HEIC, the format iPhones use: quality, Kvazaar's lossless coding, chroma\ndownsampling, the colour-profile presets and custom code points, thumbnails, alpha and\npremultiplied alpha, rotation and mirroring, tiles, 360° projection, a description,\ncompatible brands and the compact mini format.
    • \n
  • Metadata: a photo's ICC colour profile, Exif and XMP are kept as each reference\ntool keeps them: in WebP when you ask, as cwebp -metadata does (off by default, as in\ncwebp); in AVIF, JPEG XL and HEIC by default, as avifenc, cjxl and heif-enc do,\nwith each kind switchable off.
    • \n
  • Crop and resize (with -resize_mode), shared by every format.
    • \n
\n

Workflow

\n
    \n
  • Preview the result beside the original, in any format, before anything is written to\ndisk. AVIF and JPEG XL previews are decoded in Rust, so they show on every platform.
    • \n
  • Animated WebP or GIF in, animated WebP out: every frame, its timing and its loop count\nkept (or a loop count of your own), every WebP setting applied to each frame, the\nanimation encoder's keyframe spacing, mixed lossy/lossless and minimum-size options,\nand a preview of the animation first.\nA GIF converts as gif2webp converts it, across disposal methods, transparency, timing\nand loop counts, keeping its ICC profile and XMP when you ask, as\ngif2webp -metadata does.
    • \n
  • Drop files onto the window (any it cannot read are set aside), or convert a whole\nfolder, optionally with its subfolders, recreating their structure in the destination.
    • \n
  • Batch conversion with progress per file, and a Cancel button that never leaves a\nhalf-written image behind.
    • \n
  • The original and converted sizes, and the dimensions produced, reported after each\nconversion; and what an existing WebP already is: size, lossy or lossless, alpha,\nanimated.
    • \n
  • Each format keeps its own settings while you try another. Settings and the destination\nare remembered between launches, and you can save named presets of your own.
    • \n
  • Every control shows the flag it sets, with the expert ones in folding sections. The\nwhole window works from the keyboard, and every control, text and number fields\nincluded, has a name and a role a screen reader can announce. CI checks both on every\nchange: it tabs through each format's panel and checks every control is reached, and\nreads the window the way a screen reader does, over AT-SPI on Linux and UI Automation\non Windows (and with axe in the webview window). It also drops a file onto the window\nas a file manager does, through X11 on Linux and by dragging it out of Explorer on\nWindows, and converts it.
    • \n
  • Your source image is never overwritten, and every write goes through a temporary file,\nso a failed conversion cannot damage a file that was already there. Before a run, the\nwindow warns when two queued files would be written to the same name (photo.png and\nphoto.jpg both become photo.webp) and when outputs are already in the destination.\nThose are replaced, as cwebp -o would replace them, unless you turn off Replace\nexisting files in the app settings, which keeps them and skips those inputs.
    • \n
  • About and licences, in the app settings, shows the version, the edition and its\nlicence, and the licence texts and third-party notices the app carries.
    • \n
\n

Install

\n

Download the file for your computer from the\nlatest release\n(<version> is the release's version number):

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Your computerFile to download
Windows 10 or 11, 64-bitSkidbladnir_<version>_x64-setup.exe
Mac with Apple silicon (M1 or later), experimentalSkidbladnir_<version>_aarch64.dmg
Mac with an Intel processor, experimentalSkidbladnir_<version>_x64.dmg
Ubuntu 22.04 or later, Debian 12 or later, Linux Mint 21 or later, and distributions based on themSkidbladnir_<version>_amd64.deb
Fedora, openSUSE and other RPM-based distributions with glibc 2.35 or laterSkidbladnir_<version>_x86_64.rpm
Any other 64-bit Linux with glibc 2.35 or laterSkidbladnir_<version>_amd64.AppImage
Linux on 64-bit ARM (a Raspberry Pi 4 or 5 on a 64-bit system, ARM laptops and servers), glibc 2.35 or laterSkidbladnir_<version>_arm64.deb, Skidbladnir_<version>_aarch64.rpm or Skidbladnir_<version>_aarch64.AppImage, as above
\n

Not sure which edition? Download the standard edition: the files above, named\nSkidbladnir_… with an underscore. Choose the GPL edition, the same files named\nSkidbladnir-GPL_…, only if you write HEIC and want the x265 encoder: heif-enc -L\nlossless, 4:4:4 and 4:2:2 chroma, 10-bit, x265's preset and tune, its main tuning\ncontrols, and any other x265 parameter. Both are free. The licence matters only if you\nredistribute the app; Editions has the details. On ARM Linux the GPL\nedition's x265 runs without its assembly (it cannot be linked into the shared libheif\nthere), so its HEIC encodes are slower; the files are the same.

\n

The release page lists other files too; they are not installers. The .app.tar.gz files\nare for the in-app updater on a Mac. Skidbladnir-<version>-source.tar.gz is the complete\nsource with every submodule; GitHub's own "Source code" archives leave the submodules out.

\n

There are no builds for 32-bit systems or Windows on ARM. Windows on ARM can run x64\nprograms through emulation, but nobody has tried Skidbladnir there.\nStatus says which builds have been run, and how.

\n

Opening it the first time

\n

The installers are not code-signed yet, so Windows and macOS warn before they open an app\nfrom a developer they cannot identify. This is what to expect, and how to get past it.

\n

Windows. Your browser may warn that the file is not commonly downloaded. Keep it\nanyway. In Edge, open the ... (More actions) menu beside the download, choose\nKeep, then Show more > Keep anyway. In Chrome, choose Keep or Download\nsuspicious file. When you run the installer, SmartScreen shows Windows protected your\nPC: click More info, then Run anyway. Skidbladnir installs for your account\nonly, in %LOCALAPPDATA%\\Skidbladnir, without administrator rights. If Microsoft's\nWebView2 runtime is missing (Windows 10 and 11 normally have it), the installer downloads\nit. If Smart App Control is turned on (Windows Security > App & browser control),\nWindows blocks unsigned apps and offers no exception, so Skidbladnir cannot be installed\non that PC until it is signed.

\n

macOS (experimental). The macOS builds are new: CI builds them, runs the test suite\non macOS and converts to every format through the window there, but nobody has opened the\napp on a real Mac yet, so please\nreport problems. Open the .dmg,\ndrag Skidbladnir into Applications, and open it from there.

\n
    \n

  • On macOS 15 (Sequoia) or later, the first time you open it, macOS says it cannot\nverify the app. Click Done, open System Settings > Privacy & Security, scroll\ndown, click Open Anyway beside the message about Skidbladnir, and confirm.

    \n
    • \n

  • On macOS 14 or earlier, right-click (or Control-click) Skidbladnir in Applications,\nchoose Open, then choose Open again.

    \n
    • \n

  • If macOS says that Skidbladnir is damaged and can't be opened, there is no Open Anyway\nbutton. Remove the quarantine flag your browser set, in Terminal, then open it again:

    \n
    xattr -dr com.apple.quarantine /Applications/Skidbladnir.app
    • \n
\n

Linux. Both packages need glibc 2.35 or later: Ubuntu 22.04, Debian 12 or newer.

\n
    \n
  • The .deb (Ubuntu, Debian, Linux Mint): in the folder you downloaded it to, run\nsudo apt install ./Skidbladnir_<version>_amd64.deb. apt installs what it needs\n(xkbcommon and the Vulkan loader for the window, and WebKitGTK and GTK for the webview\nwindow it falls back to) and adds Skidbladnir to your applications menu.
    • \n
  • The AppImage (any distribution): run chmod +x Skidbladnir_<version>_amd64.AppImage,\nthen ./Skidbladnir_<version>_amd64.AppImage. It adds no menu entry. It needs FUSE: a\nfusermount3 or fusermount program (the fuse3 or fuse package) and /dev/fuse;\nlibfuse2 is not needed. Where FUSE is not available, run it with\n--appimage-extract-and-run.
    • \n
\n

Checking a download

\n

The release page lists a SHA-256 checksum beside each file. Work out the checksum of the\nfile you downloaded and compare the two:

\n
    \n
  • Linux: sha256sum Skidbladnir_<version>_amd64.deb
    • \n
  • macOS: shasum -a 256 Skidbladnir_<version>_aarch64.dmg
    • \n
  • Windows, in PowerShell: Get-FileHash .\\Skidbladnir_<version>_x64-setup.exe (it uses\nSHA-256 unless told otherwise)
    • \n
\n

If they differ, the file is not the one the release published: delete it and download it\nagain.

\n

Updates

\n

Once installed, Skidbladnir checks for a newer release each time it starts and offers it\nin a banner; nothing is downloaded until you click Install and restart, and every\nupdate is verified against the project's signing key before it is installed. A .deb\ninstall updates with a .deb and an .rpm install with an .rpm (each asks for your\npassword to do it); an AppImage, the\nWindows installer and the macOS app update themselves in place. Each edition updates only\nto the same edition; to switch, install the other one over it. Releases before 0.8.0\nhave no updater, so moving off them is a manual download, once.

\n

Privacy

\n

Skidbladnir has no telemetry and no analytics, and your images never leave your\ncomputer. The app makes one network request of its own: when it starts, it fetches a\nsmall update manifest (latest.json, or latest-gpl.json for the GPL edition) from\nraw.githubusercontent.com. Like any web request, that shows GitHub your IP address;\nnothing about you or your files is sent. An update is downloaded from GitHub, and\ninstalled, only when you click Install and restart.

\n

Editions

\n

The two editions are the same app except for how they write HEIC, and so for their\nlicence:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Standard (Skidbladnir_*)GPL (Skidbladnir-GPL_*)
HEIC encoderKvazaar (BSD-3-Clause)x265 (GPL-2.0-or-later)
HEIC controlsevery heif-enc option Kvazaar honoursthose, plus -L lossless, 4:4:4 and 4:2:2 chroma, 10-bit, and x265's preset, tune, TU depth, AQ, psy-rd/psy-rdoq, deblock and SAO as controls, with any other -p x265: parameter by name
HEIC checkedbyte for byte against heif-enc -e kvazaarby decoding
Licence of the buildISC; ships libheif/libde265 as a separate, replaceable LGPL-3.0 libraryGPL-3.0-or-later, as a whole
\n

The GPL edition exists for people who want x265, the HEVC encoder a default heif-enc\nbuild uses, and x265 is GPL: an app that ships it is distributed under the GPL.\nSkidbladnir's own source is ISC either way. Everything but HEIC output — WebP, AVIF,\nJPEG XL, and HEIC input — is identical in both. In both, libheif ships as a separate\nshared library beside the app, with its codec plugins switched off, so the standard\nedition can never pick up x265 from your system.

\n

Each release attaches Skidbladnir-<version>-source.tar.gz, the complete source with\nevery submodule, which is what the GPL edition's licence requires to be offered with it.\nThe Rust crates and npm packages it compiles in are pinned by Cargo.lock and\nfrontend/package-lock.json in that archive, and published unmodified on crates.io and\nnpmjs.com.

\n

What "every option" means

\n

The claim is tested, not assumed: for each format, a test runs the reference tool and\nSkidbladnir on the same input with the same options, across the whole option surface and\nacross the inputs the tool reads differently (PNG colour types and chunks, JPEG chroma\nsamplings, CMYK, metadata), and requires identical bytes. CI builds each reference from the\nsame source as the library the app links (scripts/build-reference-tools.sh). Precisely:

\n
    \n
  • Still images, and animated WebP out of an animated WebP or a GIF, as img2webp and\ngif2webp write it, including their keyframe (-kmin/-kmax), -mixed, -min_size,\n-loop and -loop_compatibility options and img2webp -d for every frame. A different\nquality or duration for each frame is not offered yet. Neither are other options\nabout more than one image: animated AVIF, HEIC or JPEG XL, image sequences,\navifenc --layered and grids assembled from several files, heif-enc with several\ninputs or -T tiled input, and cjxl from an animated GIF or APNG. They are queued in\nROADMAP.md, Phase 8.
    • \n
  • The input formats Skidbladnir reads: PNG, JPEG, TIFF, WebP, AVIF, JPEG XL, HEIC,\nGIF, PNM/PAM, PFM, PGX, Y4M, SVG, EXR and raw YUV. Raw YUV is read as cwebp -s reads it:\ninto WebP as its planes are, byte for byte, and converted to RGB by libwebp for the other\nformats, whose tools do not read it. EXR is read as cjxl reads it — linear light, its\npremultiplied alpha, chromaticities and white luminance, and its other channels kept as\nnamed extra channels — and matched byte for byte. Only cjxl reads EXR, so for the other formats it is given the\nsRGB curve, clipped at white. A JPEG XL converted to JPEG XL is decoded by libjxl as\ncjxl decodes it — its own colour, extra channels, Exif (orientation reset, as the pixels\nare turned upright), XMP, IPTC and its other boxes — and matched byte for byte, except\nthat a JPEG XL of several frames or layers is converted from its pixels.\n(heif-enc --raw cannot write a HEIC in this build, so there is nothing of it to match.) None of them reads SVG, so an SVG has no reference to\nmatch: it is drawn by resvg. cjxl reads only 8- and 16-bit PGX at the right brightness\n(a 12-bit file comes out at a sixteenth of it), so those two depths are matched byte\nfor byte and the others are read as meant. A Y4M is read by avifenc's own reader and encoded to AVIF as\nits planes are, and an 8-bit 4:2:0 one goes to HEIC as heif-enc reads it. heif-enc\nreads every other Y4M as if it were 8-bit 4:2:0 too, garbling it, so those are\nconverted to RGB (BT.601) for HEIC instead, as they are for WebP and JPEG XL, which no\nreference tool reads Y4M for. PNM is read the way cwebp reads it for WebP and the way\ncjxl reads it for JPEG XL (at its own bit depth); PFM is read as cjxl reads it, and\nonly cjxl reads PFM, so PFM to another format has no reference to match. JPEG is read\nthrough libjpeg-turbo 3.2.0, the way each tool reads it — including avifenc's and\nheif-enc's copying of a JPEG's own YCbCr planes — and PNG by libpng 1.6's rules,\nincluding cwebp's gamma correction. For WebP, 16-bit files are reduced to 8 bits the\nway cwebp's libpng or libtiff reduces them, a WebP source takes cwebp's route\nstraight to YUV, and a TIFF with premultiplied (associated) alpha is un-multiplied as\ncwebp un-multiplies it. A tool linked against a different libjpeg or libpng can\ndecode the same file differently.
    • \n
  • The same encoders, built the same way: libwebp 1.6.0, libavif 1.4.2 with libaom\n3.15.1, libjxl 0.12.0, libheif 1.23.5 with Kvazaar 2.3.2. Another version of a tool\nwrites other bytes, as it would against itself. The app's libavif and the reference\navifenc are both built without libyuv. A packaged avifenc (Homebrew, most Linux\ndistributions) usually links libyuv, which converts RGB to YUV differently, so it\nwrites other bytes even at libavif 1.4.2.
    • \n
  • Tested on Linux (x86-64 and ARM64), macOS on Apple silicon and Windows x64. CI\nbuilds the reference tools and runs the parity tests on each, so the encoders' x86 and\nARM code paths, and their MSVC builds, are each held to the tools. One exception in\nwhat is compared: for an EXR whose data window does not cover its display window, cjxl\nencodes whatever memory it was given for the uncovered pixels (libjxl 0.12.0 never writes\nthem): zero on Linux and macOS by chance, other bytes on Windows. Skidbladnir makes them zero, and those\ncases are held to cjxl reading the same image with the zeros written out. The Intel Mac build\nlinks the same sources, but no reference tool runs there, so identical bytes there are\nexpected, not tested.
    • \n
  • AVIF with libaom, avifenc's default codec: -c rav1e and -c svt are not built\nin. JPEG gain-map conversion needs avifenc built with libxml2, which it is not by\ndefault, and is not offered.
    • \n
  • HEIC with Kvazaar in the standard edition, the BSD-licensed HEVC encoder; the\ndefault heif-enc build uses x265, which is GPL, and is the GPL edition's\nencoder. heif-enc -L fails with Kvazaar, so lossless is Kvazaar's own\n(-p lossless=true), and 16-bit PNGs, which heif-enc hands Kvazaar at 10 bits and\nKvazaar refuses, are encoded from their high bytes instead. The GPL edition's x265\noutput is checked by decoding it, not byte for byte.
    • \n
  • WebP's starting settings. Until you change them, the WebP controls start from\nSkidbladnir's own long-standing defaults (-af -alpha_filter best -pass 6 -mt), not\nfrom what cwebp does with no flags. To start where cwebp starts, click Start from\ncwebp's defaults in the WebP panel (tested against cwebp given no options at all),\nthen set the flags you use.
    • \n
  • Options that change how a tool runs but not the file — verbosity, timing, benchmarks,\nprinting statistics — have no control, and neither do cwebp's dump and map outputs.
    • \n
\n

Where a tool refuses an input — cwebp a CMYK JPEG, heif-enc an RGB-coded one —\nSkidbladnir converts it anyway. Two tools misread a TIFF's transparency, and by default\nSkidbladnir reads it correctly instead, keeping the colours the file means:

\n
    \n
  • Straight (unassociated) alpha, to WebP: cwebp reads such a TIFF premultiplied,\nthrough libtiff, and darkens its semi-transparent pixels.
    • \n
  • Premultiplied (associated) alpha, to HEIC: heif-enc takes that alpha as straight,\nso its HEIC is darker where the image is semi-transparent. (Skidbladnir un-multiplies\nit, as cwebp does.)
    • \n
\n

To get the tools' results instead, turn on TIFF input → Read transparency as the\nofficial tool does: both are then reproduced byte for byte (tested against cwebp and\nheif-enc). avifenc and cjxl do not read TIFF, so AVIF and JPEG XL are unaffected.

\n

Besides the byte comparisons, the output is decoded with each format's own tools. AVIF is\ndecoded with libavif's avifdec, and the tests measure how close the pixels come back and\ncheck that transparency survives. JPEG XL is decoded with libjxl's djxl, which must\ndecode it at the right size and alpha, a lossless file exactly, and rebuild a recompressed\nJPEG byte for byte. HEIC is decoded with libheif's heif-dec and heif-info, from a\nseparate build; that is how the GPL edition's x265 output is checked. Metadata is checked\nagainst cwebp itself and with avifdec, djxl and heif-info. AVIF input is checked\nagainst avifdec across 17 colour encodings (bit depths, chroma subsampling, range,\ncolour matrices, alpha), and JPEG XL input against djxl.

\n

How it compares

\n

As each project describes itself on its own page, checked on 1 October 2026. "Not\nstated" means its page does not say, not that it cannot.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Runs asBatchWritesEncoder optionsByte parity with the reference tools
Skidbladnirdesktop app: Windows, macOS, Linux (x86-64 and ARM64)yes, files or whole foldersWebP, AVIF, JPEG XL, HEICevery option of cwebp, avifenc, cjxl and heif-enc for a still imageyes, tested: 4,270 cases (above)
Squooshweb app in the browser (installable), nothing uploadednot statedMozJPEG, WebP, AVIF, JPEG XL, OxiPNG, QOI, WebP 2 (its codec list); no HEICnot statednot stated
XnConvertdesktop app: Windows, macOS, Linux; free for private or educational use, paid in a companyyesabout 70 formats, including WebP, AVIF, JPEG XL and HEICnot statednot stated
Converseendesktop app on ImageMagick: Windows, Linux, macOS, FreeBSD; GPL-3.0yesover 100 formats, including WebP, AVIF and HEIC; JPEG XL not mentionednot statednot stated
The reference command-line toolscommand lineby scriptingone format eachall of them, by definitionthey are the reference
\n

Build from source

\n

Requires a stable Rust toolchain, Node.js and npm, nasm — libaom, libjpeg-turbo and\nthe AV1 decoder assemble their SIMD code with it, and x265 in the GPL edition (Arch:\nnasm; Debian/Ubuntu: nasm; macOS: brew install nasm; Windows: choco install nasm)\n— and CMake with a C++ compiler, which build libjxl, libavif with libaom, and\nlibjpeg-turbo from source so they are linked statically. Clone with\n--recurse-submodules: those sources are pinned under third_party/. HEIC needs libheif,\nbuilt with scripts/build-libheif.sh from the same pinned sources into build/libheif;\nwithout that build, the system libheif is linked instead. On Linux you also need\nthe development headers of webkit2gtk-4.1, and of xkbcommon (with xkbcommon-x11), xcb,\nWayland, fontconfig and FreeType for the gpui window.

\n
npm --prefix frontend install        # also installs the Tauri CLI\ncd src-tauri && ../frontend/node_modules/.bin/tauri build --no-bundle

Run it with ../frontend/node_modules/.bin/tauri dev from src-tauri/, which starts the\nNuxt dev server and the window together. Release builds must go through tauri build\nrather than cargo build --release: the custom-protocol feature that embeds the\nfrontend is only enabled by the former.

\n

Without CMake, the build links the system libjxl, libavif and libjpeg through\npkg-config instead, and says so. Set SKIDBLADNIR_LIBJXL, SKIDBLADNIR_LIBAVIF or\nSKIDBLADNIR_LIBJPEG to vendored to insist on the static build, or to system to insist\non the system library.

\n

The GPL edition is the gpl feature (or full, every optional feature), with a libheif\nbuilt around x265 into build/libheif-gpl. The build refuses to link either edition\nagainst the other's libheif.

\n
scripts/build-libheif.sh --edition gpl\ncd src-tauri && ../frontend/node_modules/.bin/tauri build --features gpl --config tauri.gpl.conf.json

tauri.gpl.conf.json (and on Windows tauri.gpl.windows.conf.json too) bundles x265's\nlibheif, the GPL edition's notices and the GPL text, and points the updater at the GPL\nedition's releases.

\n

The encoders are the libraries themselves, linked into the binary — there is no command-line\ntool to download and no subprocess.

\n

Checking a build

\n
cargo test --workspace     # includes the encoder-parity tests\nscripts/smoke-test.sh      # drives the webview window: renders, IPC, a real conversion\nscripts/a11y-audit.sh      # axe-core against the live webview window\nscripts/gpui-smoke.sh      # opens the gpui window and converts to every format

The parity tests compare each encoder with its reference tool and need them built from\nthe same sources as the libraries the app links, reading JPEG through the same\nlibjpeg-turbo: scripts/build-reference-tools.sh builds cwebp, img2webp, gif2webp,\navifenc, cjxl and heif-enc and prints the SKIDBLADNIR_REFERENCE_* variables that\npoint the tests at them. The decoding checks use libavif's avifdec and avifenc,\nlibjxl's djxl and libheif's heif-dec and heif-info from PATH. Missing tools make\nthe tests say so and skip rather than pass quietly; set SKIDBLADNIR_REQUIRE_PARITY=1 to turn a skip\ninto a failure.

\n

The two scripts need tauri-driver (cargo install tauri-driver) and WebKitWebDriver\n(webkit2gtk-driver on Debian/Ubuntu, webkitgtk-6.0 on Arch). They skip, loudly, when\nthose are missing.

\n

Status

\n

Skidbladnir is a Rust + Tauri 2 app for Windows, Linux and macOS, and it builds and\npasses its tests on all three in CI. Since 1.3.0 its window is drawn natively with\ngpui (crates/skidbladnir-gpui): the same window, control for\ncontrol, in about a third of the memory and starting about 30% sooner. The Nuxt +\nTailwind webview window it replaces is still built in. It opens when the app is started\nwith SKIDBLADNIR_UI=webview, or by itself when the gpui window cannot start (on a\nmachine with no GPU driver it can use, say). Releases are\nstable from 1.0.0; the ones before it were prereleases. The work queue lives in\nROADMAP.md.

\n

Skidbladnir used to be an Electron app for Windows. That app has been retired: its last\nbinary release is v0.4.3\n(2019), and its last source is the\nelectron-final tag.

\n

Known gaps:

\n
    \n
  • macOS is experimental. The .dmgs (Apple silicon and Intel) are built by CI, and\nthe test suite runs on macOS, where CI also opens the window and converts to every\nformat with it, but nobody has yet opened the app on a real Mac. The\nstandard edition's Windows installer is installed and exercised by CI on every change,\nand the Linux app (both editions) is launched and driven by CI on every change, though\nnot from its .deb or AppImage. Each release's .deb is installed by CI on Ubuntu 22.04\nand Debian 12, and its .rpm on the latest Fedora, on x86-64 and ARM64 alike, and\nchecked to find every library it needs; the ARM64 app itself has not been launched. The\npublished 0.5.0 .deb and AppImage were run through that release's smoke test by hand.
    • \n
  • Nobody has used Skidbladnir with a screen reader yet (Orca, Narrator, NVDA or\nVoiceOver). What one is given is checked automatically on Linux and Windows, not on\nmacOS. Before 1.4.1 the window's text and number fields (sizes, crops, exact values)\nwere missing from it entirely.
    • \n
  • The app is not code-signed on any platform;\nOpening it the first time shows how to get past each\nsystem's warning.
    • \n
  • An AVIF, JPEG XL or HEIC encode cannot be cancelled mid-file: none of those encoders\nreports progress, so Cancel takes effect when the current file finishes (which is then\ndiscarded, not written).
    • \n
  • An animated AVIF, JPEG XL or HEIC is read as one still image: its primary image or\nfirst frame.
    • \n
  • Animated AVIF, JPEG XL and HEIC are not written (see below), so an animation's metadata\nis kept in animated WebP only: a GIF's ICC profile and XMP as gif2webp -metadata keeps\nthem, an animated WebP's ICC profile, Exif and XMP as webpmux -set would set them.
    • \n
  • An animation (animated WebP or GIF) converts to WebP only. AVIF, JPEG XL and HEIC\noutput refuse it by name rather than keeping just the first frame, because those\nencoders write still images here. A target size or PSNR applies to each frame of an\nanimation, not to the whole file.
    • \n
  • JPEG XL files open in Safari, and in Firefox and Chrome as each enables it by default;\nuntil then, most web pages cannot show them. The long-promised JPEG 2000 was dropped\nas a goal; it has no momentum outside medical and archival imaging.
    • \n
  • The standard edition writes HEIC 8-bit 4:2:0 only (Kvazaar), so its lossless coding is\nlossless after the conversion to 4:2:0, as with heif-enc; -L lossless, 4:4:4 and\n10-bit take the GPL edition. Neither writes 12-bit HEIC.
    • \n
  • HEVC patents. HEIC is HEVC (H.265) inside HEIF, and HEVC is covered by patents\nlicensed through patent pools such as Access Advance and Via LA. Skidbladnir includes\nopen-source HEVC software: Kvazaar (x265 in the GPL edition) to write HEIC, and libde265\nto read it. The project holds no HEVC patent licence and passes none on. Whether you\nneed one depends on where you are and what you do with the files; this is not legal\nadvice.
    • \n
\n

Acknowledgements

\n

Skidbladnir is a window around other people's encoders, and is only as good as they are:\nlibwebp (the WebM project, Google),\nlibavif and\nlibaom (the Alliance for Open Media),\nlibjxl (the JPEG XL authors),\nlibheif and\nlibde265 (struktur AG),\nKvazaar (Ultra Video Group, Tampere University),\nx265 (MulticoreWare) and\nlibjpeg-turbo (which includes the work of the Independent\nJPEG Group). The decoders are rav1d through\navif-decode, and\njxl-oxide. The window is\nTauri, Nuxt, Nuxt UI and\nTailwind CSS, set in Fira Code,\nwith icons from the collections THIRD-PARTY-NOTICES.md credits.

\n

Skidbladnir is an independent project. It is not affiliated with, sponsored or endorsed\nby Google, the Alliance for Open Media, the JPEG committee, Apple, Microsoft, or any of\nthe projects above. WebP, AVIF, JPEG XL, HEIC and the other names here are used only to\nsay which formats and tools it works with; they belong to their owners.

\n

License

\n

Skidbladnir's own code is ISC (LICENSE), as declared in the workspace\nCargo.toml, apart from code adapted from the encoders, which keeps its licence:\ncrates/skidbladnir-encode/native/avif_shim.c carries portions of libavif's avifenc\nunder BSD-2-Clause.

\n

The standard edition's own code is ISC; the libraries it ships keep their licences,\nincluding LGPL-3.0 for libheif and libde265, which ship as a separate shared library you\nmay replace with your own build. The GPL edition's builds include x265, so each of them\nis distributed as a whole under the GNU GPL, version 3 or later\n(LICENSES/GPL-3.0.txt); see Editions.

\n

The native libraries each edition ships, the JavaScript packages in its webview window and\nthe Rust crates compiled into it, with their licences, are listed in\nTHIRD-PARTY-NOTICES.md and\nTHIRD-PARTY-NOTICES-GPL.md, and every installer carries its\nown edition's copy with its licence texts.

\n", "releases": [ + { + "tag": "v1.4.4", + "title": "Skidbladnir 1.4.4", + "url": "https://github.com/basic-automation/Skidbladnir/releases/tag/v1.4.4", + "publishedAt": "2026-10-13T14:32:48Z", + "prerelease": false + }, { "tag": "v1.4.3", "title": "Skidbladnir 1.4.3", @@ -226,122 +233,115 @@ "url": "https://github.com/basic-automation/Skidbladnir/releases/tag/v1.4.0", "publishedAt": "2026-10-03T05:14:28Z", "prerelease": false - }, - { - "tag": "v1.3.0", - "title": "v1.3.0 — a native window: faster to start, a third of the memory", - "url": "https://github.com/basic-automation/Skidbladnir/releases/tag/v1.3.0", - "publishedAt": "2026-10-02T19:06:47Z", - "prerelease": false } ], "latestRelease": { - "tag": "v1.4.3", - "url": "https://github.com/basic-automation/Skidbladnir/releases/tag/v1.4.3", - "publishedAt": "2026-10-12T18:15:50Z" + "tag": "v1.4.4", + "url": "https://github.com/basic-automation/Skidbladnir/releases/tag/v1.4.4", + "publishedAt": "2026-10-13T14:32:48Z" }, "download": { - "tag": "v1.4.3", - "url": "https://github.com/basic-automation/Skidbladnir/releases/tag/v1.4.3", - "publishedAt": "2026-10-12T18:15:50Z", + "tag": "v1.4.4", + "url": "https://github.com/basic-automation/Skidbladnir/releases/tag/v1.4.4", + "publishedAt": "2026-10-13T14:32:48Z", "prerelease": false, "assets": [ { - "name": "Skidbladnir-GPL_1.4.3_aarch64.AppImage", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_aarch64.AppImage", - "size": 108255752 + "name": "Skidbladnir-GPL_1.4.4_aarch64.AppImage", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_aarch64.AppImage", + "size": 108263944 }, { - "name": "Skidbladnir-GPL_1.4.3_aarch64.dmg", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_aarch64.dmg", - "size": 23595111 + "name": "Skidbladnir-GPL_1.4.4_aarch64.dmg", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_aarch64.dmg", + "size": 24019543 }, { - "name": "Skidbladnir-GPL_1.4.3_aarch64.rpm", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_aarch64.rpm", - "size": 36192244 + "name": "Skidbladnir-GPL_1.4.4_aarch64.rpm", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_aarch64.rpm", + "size": 36248954 }, { - "name": "Skidbladnir-GPL_1.4.3_amd64.AppImage", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_amd64.AppImage", - "size": 110217720 + "name": "Skidbladnir-GPL_1.4.4_amd64.AppImage", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_amd64.AppImage", + "size": 110254584 }, { - "name": "Skidbladnir-GPL_1.4.3_amd64.deb", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_amd64.deb", - "size": 35993580 + "name": "Skidbladnir-GPL_1.4.4_amd64.deb", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_amd64.deb", + "size": 36041638 }, { - "name": "Skidbladnir-GPL_1.4.3_arm64.deb", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_arm64.deb", - "size": 36191822 + "name": "Skidbladnir-GPL_1.4.4_arm64.deb", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_arm64.deb", + "size": 36249084 }, { - "name": "Skidbladnir-GPL_1.4.3_x64-setup.exe", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_x64-setup.exe", - "size": 15699101 + "name": "Skidbladnir-GPL_1.4.4_x64-setup.exe", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_x64-setup.exe", + "size": 15713749 }, { - "name": "Skidbladnir-GPL_1.4.3_x64.dmg", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_x64.dmg", - "size": 26327252 + "name": "Skidbladnir-GPL_1.4.4_x64.dmg", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_x64.dmg", + "size": 26705236 }, { - "name": "Skidbladnir-GPL_1.4.3_x86_64.rpm", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir-GPL_1.4.3_x86_64.rpm", - "size": 35990900 + "name": "Skidbladnir-GPL_1.4.4_x86_64.rpm", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir-GPL_1.4.4_x86_64.rpm", + "size": 36040394 }, { - "name": "Skidbladnir_1.4.3_aarch64.AppImage", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_aarch64.AppImage", - "size": 106453512 + "name": "Skidbladnir_1.4.4_aarch64.AppImage", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_aarch64.AppImage", + "size": 106490376 }, { - "name": "Skidbladnir_1.4.3_aarch64.dmg", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_aarch64.dmg", - "size": 21366327 + "name": "Skidbladnir_1.4.4_aarch64.dmg", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_aarch64.dmg", + "size": 21756777 }, { - "name": "Skidbladnir_1.4.3_aarch64.rpm", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_aarch64.rpm", - "size": 34056887 + "name": "Skidbladnir_1.4.4_aarch64.rpm", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_aarch64.rpm", + "size": 34111467 }, { - "name": "Skidbladnir_1.4.3_amd64.AppImage", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_amd64.AppImage", - "size": 108345848 + "name": "Skidbladnir_1.4.4_amd64.AppImage", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_amd64.AppImage", + "size": 108407288 }, { - "name": "Skidbladnir_1.4.3_amd64.deb", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_amd64.deb", - "size": 33518070 + "name": "Skidbladnir_1.4.4_amd64.deb", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_amd64.deb", + "size": 33572472 }, { - "name": "Skidbladnir_1.4.3_arm64.deb", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_arm64.deb", - "size": 34058542 + "name": "Skidbladnir_1.4.4_arm64.deb", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_arm64.deb", + "size": 34110724 }, { - "name": "Skidbladnir_1.4.3_x64-setup.exe", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_x64-setup.exe", - "size": 14934443 + "name": "Skidbladnir_1.4.4_x64-setup.exe", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_x64-setup.exe", + "size": 14975921 }, { - "name": "Skidbladnir_1.4.3_x64.dmg", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_x64.dmg", - "size": 23930505 + "name": "Skidbladnir_1.4.4_x64.dmg", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_x64.dmg", + "size": 24301798 }, { - "name": "Skidbladnir_1.4.3_x86_64.rpm", - "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.3/Skidbladnir_1.4.3_x86_64.rpm", - "size": 33516595 + "name": "Skidbladnir_1.4.4_x86_64.rpm", + "url": "https://github.com/basic-automation/Skidbladnir/releases/download/v1.4.4/Skidbladnir_1.4.4_x86_64.rpm", + "size": 33571437 } ] } }, "enlil": { "repo": "enlil", - "fetchedAt": "2026-10-13T05:41:35.359Z", + "fetchedAt": "2026-10-14T04:35:15.574Z", "description": "Bare-Metal Mesh Hypervisor", "htmlUrl": "https://github.com/basic-automation/enlil", "homepage": null, @@ -353,9 +353,9 @@ "license": "MIT", "defaultBranch": "master", "createdAt": "2026-03-26T14:25:45Z", - "pushedAt": "2026-10-12T23:24:34Z", + "pushedAt": "2026-10-14T03:52:17Z", "archived": false, - "readmeHtml": "

Enlil

\n
\n

Enlil — the Sumerian god who separated heaven from earth, ruled the space between, and assigned domains to lesser gods.

\n
\n

A portable, bare-metal-capable Type-1 hypervisor written in Rust that turns a single\nx86 desktop into multiple transparent virtual PCs — with granular peripheral routing, GPU\nsharing, and a path toward pooling hardware across many physical machines.

\n
    \n
  • Language: 100% Rust (no_std for the core, std for tooling)
    • \n
  • Foundation: the RustVMM crate ecosystem
    • \n
  • Target hardware: x86-64 desktops with Intel VT-x/VT-d or AMD-V/AMD-Vi (ARM & RISC-V planned)
    • \n
  • Boot model: a UEFI application on a USB stick — non-destructive, coexists with your existing OS
    • \n
  • License: MIT
    • \n
  • Security: pre-1.0 research software — do not run untrusted guests. See SECURITY.md.
    • \n
\n
\n

What is Enlil?

\n

Most virtualization runs N guests on 1 host. Enlil's near-term goal is to make a single\nmachine feel like several independent PCs — each guest pinned to its own cores and memory,\neach seeing what looks like real, dedicated hardware — and to do it transparently enough\nthat a guest OS (including Windows, with anti-VM-detection hardening) cannot tell it is\nvirtualized.

\n

The longer-term vision is logical machines over a physical pool: Enlil mediates both\ndirections of every hardware interaction (guest→hardware traps and hardware→guest\ninterrupts) and can route them across a pool of nodes. That turns physical machines into\nstateless hardware providers and each guest into a logical machine defined only by a\nresource manifest and a routing table — hardware disaggregation underneath unmodified\noperating systems. See the Core Model\nbelow for the full picture.

\n

A defining usability constraint shapes the whole project: Enlil must be testable without\nmodifying your system. It boots from a USB stick as a standard UEFI application\n(/EFI/BOOT/BOOTX64.EFI), loads its config and per-guest virtual firmware from that stick,\nand leaves every existing drive and bootloader untouched. Unplug it and reboot to return to\nbare metal instantly.

\n
┌─────────────┐  ┌─────────────┐  ┌─────────────┐\n│  Guest OS 1 │  │  Guest OS 2 │  │  Management  │\n│ (Windows)   │  │  (Linux)    │  │   Console    │\n├─────────────┤  ├─────────────┤  ├─────────────┤\n│ Virtual HW  │  │ Virtual HW  │  │  Config API  │\n│ vCPU · vGPU │  │ vCPU · vGPU │  │  USB Route   │\n│ vUSB · vNIC │  │ vUSB · vNIC │  │  GPU Policy  │\n├─────────────┴──┴─────────────┴──┴─────────────┤\n│                  ENLIL CORE                    │\n│   CPU/Mem Sched · GPU Arbiter · USB Router ·   │\n│            Storage / Net Manager               │\n├────────────────────────────────────────────────┤\n│           UEFI Firmware (boot payload)         │\n├────────────────────────────────────────────────┤\n│                Physical Hardware               │\n└────────────────────────────────────────────────┘

Each guest gets its own virtual UEFI (OVMF) and its own bootloader inside the VM — Enlil\nsits below everything and never replaces GRUB or the Windows Boot Manager.

\n
\n

Core Model — Logical Machines over a Physical Pool

\n

The near-term product (transparent virtual PCs on one machine, described in What is Enlil?) is one point on a much larger design. Enlil mediates both directions of every hardware interaction — the guest→hardware path (VM-exits, MMIO/PIO traps, hypercalls, VirtIO kicks) and the hardware→guest path (physical IRQs, DMA completions, input events, captured on the node that owns the device and injected as virtual interrupts on the node running the guest). Because both directions pass through Enlil, the mapping from a guest to the hardware it runs on is arbitrary and may cross machine and network boundaries.

\n

That turns classic virtualization inside out. Instead of N guests on 1 host, Enlil aims for N guests on M hosts, fully composable: each physical machine becomes a stateless hardware node (a resource provider), and each guest becomes a logical machine defined only by a resource manifest plus a routing table. It is hardware disaggregation underneath unmodified operating systems — the ambition behind CXL, RDMA device pools, and LegoOS, but working transparently beneath stock Windows and Linux rather than requiring a custom OS.

\n

Three physical laws bound what such a pool can actually do, and they shape the whole design.

\n

Interconnect latency sets the granularity of sharing

\n

How tightly two nodes can share resources is governed by the latency of the link between them. Local DRAM is ~80 ns; a WAN round-trip is ~20–40 ms — roughly 100,000× slower — so a vCPU can never run against RAM that lives thousands of miles away.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
InterconnectAdded latencyWhat can be pooled transparently
On-board PCIe / CXLsub-µs (~200–400 ns)almost anything, including memory as a NUMA tier and live devices
RDMA, same datacenter~1–5 µsstorage, NIC, GPU-compute, cold-page memory — not a hot vCPU's RAM
WAN, 1000s of miles~20–40 ms RTTcoarse/async only: remote storage, streamed I/O, whole-guest migration, replication
\n

The rule that falls out: a single kernel's tightly-coupled hot CPU+RAM working set stays on one node (or one CXL-NUMA domain). Across distance you move or replicate a guest — you don't share its hot state. Everything else (devices, storage, GPU offload, spare capacity) is poolable with latency-aware placement. To make this practical, every virtual device is built as a front-end/back-end pair behind a pluggable transport (local | CXL | RDMA | network), so remoting a device is a transport swap rather than a rewrite.

\n

ISA sets the boundary of a logical machine

\n

The latency law has a hardware-architecture twin: a single kernel's execution cannot span an ISA boundary. Page-table formats, the interrupt model, atomics, and the memory model are all ISA-fixed, and no machine description a stock OS accepts (ACPI or device tree) can express mixed-ISA SMP. So every logical machine carries a placement key — (ISA, vendor, feature baseline) — frozen at creation; its vCPUs run natively only on matching nodes, and that key defines its live-migration domain. Foreign-ISA nodes can still serve every other lane (device back-ends, memory tiers, fabric work units are ISA-blind).

\n

A guest may optionally run via binary translation, expressed as a per-guest execution mode:

\n
    \n
  • transparent — vCPUs run natively on matching nodes; remote nodes contribute device back-ends only. This is the full-fidelity product.
    • \n
  • hybrid — native vCPUs plus a translated compute device (an opt-in paravirtual accelerator); the native surface stays fully stealthed.
    • \n
  • mesh — all vCPUs translated (JIT); location- and ISA-free, but a compatibility mode only, and explicitly detectable.
    • \n
\n

Each guest manifest declares the floor it will accept (e.g. require = transparent … allow = mesh) and the pool's composition decides what is achievable — Windows-for-ARM on an all-x86 pool admits only in mesh mode, if at all. Modes are switchable at runtime without a guest reboot. Crucially, translation is a compatibility tool, never a transparency one: per-block overhead and timing artifacts defeat the anti-detection guarantees, so the translated layer is always presented to the guest as a device, never as extra CPUs.

\n

Trust domains: your own pool vs. federating with others

\n

Latency and ISA decide what can be pooled and where it runs natively. A third, orthogonal axis — the trust domain — decides how compute must be protected once a pool spans more than one owner.

\n
    \n
  • An "Enlil device" = every node under one owner. Your phone, desktop, and laptop join into a single pool that presents as one logical-machine device. Inside this boundary compute is trusted and shared in the clear — plaintext memory tiers, zero-copy fabric, native vCPU placement — exactly the Core Model above.
    • \n
  • Federating across owners = untrusted compute. Two Enlil devices can share capacity (borrow a friend's idle GPU or cores), but across that boundary a node you don't own must never see plaintext or be trusted for integrity. That is the blind-compute regime: secret-shared / MPC / ZK-verified work units, backed by attestation that the peer runs genuine Enlil.
    • \n
\n

Trust is orthogonal to latency — a node can be near-but-untrusted (a friend's LAN) or far-but-trusted (your own VPS) — so the placement key gains a trust dimension on top of (latency class, ISA, vendor, feature baseline). Each logical machine declares which trust domains may serve it: a sensitive guest pins to trust = owner-only, while fungible, decomposable work may spill to federated-blind capacity. This is what extends N guests on M hosts to M hosts owned by different people.

\n

Transparency and guest-OS feasibility

\n

Transparency is per-guest-family: Enlil presents whatever machine model each OS expects (PC/UEFI, ARM SoC, Apple platform) and defeats that family's specific VM-detection vectors, generalizing "transparent virtual PC" to a transparent virtual machine, platform-shaped per guest. How far that reaches today:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
GuestFeasibility
Linux, *BSD (x86)✅ near-term
Windows (x86)✅ in scope, with VM-detection hardening
Android (x86)✅ near-term
Android (ARM)⏳ needs the ARM backend; Play Integrity is hardware-attested
macOS⚠️ Apple hardware only — license + SMC/board-id, a major transparency lift
iOS❌ research-only — Apple-signed boot chain + Secure Enclave, not transparently virtualizable on non-Apple hardware
\n
\n

Status

\n

Enlil is pre-alpha and under active, incremental development. Work happens on a\nKVM-backed development path on a Linux host first; the bare-metal VMX/SVM backend comes\nlater (this is the same pragmatic strategy Firecracker and Cloud Hypervisor used).

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AreaState
Platform layer + custom std target (x86_64-unknown-enlil)✅ Implemented & tested (enlil-platform, enlil-std)
HAL portability trait (HypervisorBackend)✅ Defined (enlil-hal)
KVM-backed VMM: single & multi-guest, CPU/memory partitioning✅ Implemented on Linux (enlil-core)
Virtual device layer (VirtIO net/block, interrupts, timers, PS/2, HDA, storage)✅ Implemented (enlil-devices)
Managed vCPU run loop: stealth stack + per-entry platform-timer cadence & LAPIC TSC-deadline firing✅ Implemented on Linux (enlil-core, StealthRunLoop)
Guest-boot orchestrator: config→GuestRuntime→run, bzImage kernel load (setup-header/boot_params/initrd/E820, real·protected·long-mode entry), per-guest CPUID+vTPM+LBR transparency, guest-RAM I/O — driven by the enlil-run binary✅ Implemented on Linux (enlil-core, orchestrator), KVM-verified end to end
Suspend/resume: vCPU state snapshot/restore + automatic ACPI S3 resume — a guest that commits a suspend is resumed in place (FACS waking-vector discovery via the RSDP→XSDT→FADT→FACS walk, re-entry at the waking vector) with no teardown✅ Implemented on Linux (enlil-core, KVM-verified)
USB peripheral routing (virtual xHCI, TRB-level; config-driven [usb.routing] rules assembled into a controller-validated routing registry; Linux sysfs host enumeration + cadence-throttled hot-plug polling; console USB-tab + guest-list view-models/controllers)🚧 Substantially implemented
Windows transparency (ACPI/SMBIOS synthesis incl. S3/S4/S5 sleep states, CPUID identity+topology/timing/LBR stealth, per-guest vTPM with persistent NV state, independent endorsement keys & name-derived seeds; NIC-OUI transparency in config + safe MAC synthesis; anti-detection: IET APERF/MPERF divergence + CPUID hypervisor-tell decoders — with the hypervisor bit, empty 0x40000000 vendor leaf, and preserved Invariant-TSC all KVM-verified from inside a real guest)🚧 Building blocks implemented
Bare-metal UEFI boot (Phase 6)🚧 The bare-metal kernel now boots and brings the machine up under its own control — proven nightly under QEMU+OVMF. The UEFI payload's efi_main collects the ACPI RSDP + GOP framebuffer + final UEFI memory map into a BootHandoff, calls ExitBootServices(), prints its liveness banner to COM1 serial, and transitions into the enlil kernel, which then, with no firmware left: walks the handed-off memory map and reports usable RAM; installs its own heap (a switching global allocator: firmware pool → the kernel's linked-list heap in the largest conventional region) and proves a live allocation; loads its own IDT with exception handlers and self-tests it by taking an int3; enables the local APIC in x2APIC mode; enables AMD SVM (EFER.SVME) and programs the VM_HSAVE_PA host state-save area; runs a guest through a real #VMEXIT dispatch engine — it allocates the guest its own isolated 2 MiB RAM window and maps guest-physical 0 onto it (a non-identity NPT, so the guest sees its RAM at GPA 0 while it lives at a disjoint system-physical base — LOCKED PRINCIPLE 5), writes a real-mode program into it, and VMRUNs in a loop that routes every #VMEXIT through the HAL's arch-neutral VmExit model with a guest-GPR save/restore shell around each entry (VMRUN carries only RAX/RSP/RIP/RFLAGS; the shell swaps the other 14 registers so the guest keeps state across an intercepted-and-resumed instruction). The loop emulates the guest's intercepted instructions: CPUID answered with a stealthed host result — the guest reads CPUID.1:ECX[31] (hypervisor-present) as 0, so it cannot detect enlil in-guest (LOCKED PRINCIPLE 1); port I/O decoded + captured; RDMSR answered with a stealth value and WRMSR shadowed per-guest and read back (the write never reaches host hardware); a nested page fault on an unmapped GPA is demand-mapped (fresh frame + NPT leaf) and the access re-executes; and a native arithmetic loop runs a taken branch to completion with no #VMEXIT — near-native guest execution — before the guest HLTs. The run shell brackets VMRUN with VMSAVE/VMLOAD so the guest's FS/GS/TR/LDTR + SYSENTER state is swapped in and out (a guest using segmentation/syscalls is safe); enlil delivers interrupts into a guest and the guest handles and returns from them: an EVENTINJ-injected interrupt vectors through the guest's real-mode IVT to a handler that does work in guest RAM (which the hypervisor reads back through the NPT window) and IRETs to resume the interrupted code; the same round-trip runs in 64-bit long mode through a real guest GDT + 64-bit IDT interrupt gate + IRETQ (the way a real x86-64 OS handles interrupts); a maskable virtual interrupt posted via the VMCB INT_CONTROL (V_INTR) is correctly held off while the guest masks interrupts (IF=0) and delivered only when it STIs; and enlil forcibly preempts a guest spinning in an infinite loop that never yields by posting that virtual interrupt — pure time-slicing, the mechanism a scheduler quantum uses to reclaim a CPU. That preemption now runs off the real hardware clock: with the VMCB's physical-interrupt (INTR) intercept armed, a host LAPIC timer that fires while a guest is spinning takes an INTR #VMEXIT straight back into enlil — the host reclaims the CPU from a non-cooperative guest on a genuine time base rather than a pre-posted interrupt. enlil then delivers that tick into the guest: it services and EOIs the physical interrupt, re-posts it to the guest as a virtual interrupt, and resumes without advancing RIP, so the guest vectors through its own long-mode IDT to its own handler and returns — the full path a scheduler tick takes from a host hardware clock into a running guest OS. enlil also traps a guest's own exceptions: it arms the VMCB exception-intercept bitmap so a guest #UD (or any vector) takes a #VMEXIT, then re-delivers the fault to the guest's own IDT/IVT (carrying the error code for the vectors that push one) — the guest handles its own fault while enlil observes it. It runs a full interrupt round-trip — an injected interrupt's handler runs, IRETs, and the guest resumes past the injection point (the inject → handle → IRET → resume cycle a virtual timer tick needs). And it write-protects guest memory in the NPT: a guest store to a read-only page takes a present+write nested page fault enlil records and grants (then the store re-executes) — the dirty-page / copy-on-write signal live migration builds on, while reads and instruction fetches from the same page still succeed. enlil services a VMMCALL paravirt hypercall (result returned in the guest RAX); and a full 64-bit long-mode guest runs with paging on, its own CR3 page-table walk resolved through the NPT (the mode a real x86-64 OS boots in). All boot-verified nightly on real nested SVM. The kernel also loads its own GDT + TSS with three Interrupt Stack Table stacks (the #DF, #PF, and #GP handlers each run on their own known-good stack, every IST-switch self-tested), arms the LAPIC timer in both one-shot and TSC-deadline mode (the precise-preemption clock a scheduler quantizes on — it even arms a 2 ms deadline from a nanosecond request and confirms via its own clock that it fired on time), calibrates the TSC against the fixed-rate PIT (~4.2 GHz measured) and turns it into a monotonic nanosecond clock with a busy-sleep (verified by a 5 ms sleep), installs a per-CPU GS-base TLS block, switches to its own identity page tables (off the firmware's, span derived from the memory map + framebuffer) with the low 2 MiB rendered at 4 KiB granularity and virtual address 0 left unmapped as a null-page guard (a stray kernel null-pointer dereference faults into the #PF handler instead of silently touching page 0), and can refine any other 2 MiB region of that live map down to 4 KiB pages, keeping the bulk huge-page-granular while a chosen region is managed page by page. It uses that to run on a stack it owns with a guard page: the whole rest of bring-up executes on kernel-allocated memory instead of the firmware's boot-services stack, with the page below it left unmapped so a stack overflow takes a diagnosable #PF (on its own IST stack) rather than silently corrupting its neighbour — the guard is proven absent by walking the live page tables, not by touching it. It brings up the other CPUs: the BSP finds a firmware-conventional page below 1 MiB for the AP startup trampoline (discovered from the real memory map, not a hardcoded address), installs a 16-bit real-mode trampoline, and wakes each application processor from the MADT inventory with INIT-SIPI-SIPI at the SDM's timings, each AP reporting in on a shared counter before a bounded timeout. It self-tests a spinlock primitive, and draws both a boot indicator and an 8×8-font text banner to the GOP framebuffer (pixel-readback verified). It discovers real hardware from the firmware tables the UEFI stage handed off: ACPI RSDP→XSDT→MADT (enabled-CPU count plus each processor's APIC ID — the SMP AP inventory, booted under -smp 2), MCFG (the PCIe ECAM window), and DMAR/IVRS — for VT-d it parses the DMAR body into each DMA-remapping hardware unit (register block, PCI segment, catch-all flag) and decodes that unit's device scopes (which endpoints, bridges, I/O APICs and HPETs it governs — what can be placed in a per-guest DMA domain), verified nightly against an emulated VT-d IOMMU's real firmware table. It enumerates PCI both via the legacy 0xCF8/0xCFC mechanism (host-bridge identity, device classes, MSI-X capability, and the first memory BAR — base and size via the standard all-ones write-probe, e.g. the real VGA BAR sized at 16 MiB, plus a summary of every memory BAR on the bus — their count and total MMIO footprint, the window the hypervisor must account for in config-space routing and passthrough) and via ECAM MMIO (all buses, extended config space). Each step is asserted on serial by the headless QEMU+OVMF harness (scripts/qemu-boot-test.sh, KVM-accelerated, nested +svm). The HAL carries the decode + region seams both backends need: enlil-hal::vmx (Intel VMX capability MSRs, VMCS field/exit encodings, VMXON/VMCS region init) and enlil-hal::svm (AMD SVM CPUID/VM_CR gates, the VMCB control/save-area layout, #VMEXIT decode), plus enlil-hal::region owning the 4 KiB-aligned VMCS/VMXON/VMCB/host-save pages, enlil-hal::svm's VMCB programming layer (program_minimal_hlt_guest + segment/exit-field accessors) and enlil-hal::npt's nested-page-table identity-map builder — all built for the x86_64-unknown-enlil kernel target. The kernel now links enlil-platform directly (Phase 1.2 — the crate is no_std under platform-baremetal, so its host-agnostic modules cross-compile for the bare-metal target): it drives enlil-platform's work-stealing scheduler (four tasks submitted out of order run Critical→Low on real hardware, their closures heap-boxed through the kernel's own allocator across the crate boundary), and drives that scheduler's quantum from the LAPIC TSC-deadline timer — one dispatch per real hardware tick, each quantum armed as a nanosecond deadline, so the bare-metal scheduler advances on the same clock a preemptive OS quantizes on, builds a MemoryMap from the real firmware map and carves a disjoint DMA / heap / per-guest RAM plan (cross-checked against the kernel's own memory walk — LOCKED PRINCIPLE 5) after marking the already-installed kernel heap as in-use, so the planner can never hand a guest memory the hypervisor is allocating from — and guests now run out of that planned region rather than the kernel heap, so a guest's nested mapping cannot reach the hypervisor's own allocator. It times a busy-sleep with enlil-platform's Instant monotonic clock — all boot-verified, so the boot kernel consumes shared platform primitives instead of re-deriving them. Still landed: discover::build_device_tree assembles a HostDeviceTree from the ACPI tables (CPUs with NUMA nodes via MADT + SRAT, per-node RAM ranges and distances via SRAT/SLIT, a full MCFG-ECAM PCI walk incl. BARs/xHCI/capabilities/SR-IOV, DMAR/IVRS IOMMU kind), plus MemoryMap carve/plan from E820 & UEFI maps and x86-64 identity page-table builders. Next: bringing each started AP from real mode into long mode with its own guarded stack and per-CPU state (multi-vCPU guests follow), programming the discovered IOMMU units' root/context tables for per-guest DMA isolation, and booting a real distro / privileged service VM
GPU sharing, compute fabric, ARM/RISC-V, multi-machine mesh⏳ Planned (Phases 7–11)
\n
\n

Honest caveat: the RustVMM/KVM dependencies in enlil-core are gated to\ntarget_os = "linux", and live guest-boot/integration tests require KVM (nested\nvirtualization). On other hosts the logic-level unit tests still build and run, but a real\nguest boot does not. Wiring enlil-core and enlil-devices fully through enlil-hal is\nin progress. See ROADMAP.md for exactly what is built vs. planned.

\n
\n
\n

Architecture

\n

Enlil is a Cargo workspace of focused crates layered over a custom platform abstraction. The\nkey idea — platform-first — is that every crate above the platform layer writes ordinary\nRust (Vec, HashMap, Arc<Mutex<T>>, async/await), whether the final target is Linux\nor bare metal. The platform crate absorbs all the no_std complexity.

\n
┌─────────────────────────────────────────────────────────────────┐\n│                          Enlil Crates                           │\n│  ┌──────────┐ ┌───────────┐ ┌────────────┐ ┌───────────────┐   │\n│  │enlil-core│ │enlil-mgmt │ │enlil-config│ │ enlil-devices │   │\n│  └────┬─────┘ └─────┬─────┘ └─────┬──────┘ └──────┬────────┘   │\n│       │             │             │               │            │\n│  ┌────┴─────────────┴─────────────┴───────────────┴────────┐   │\n│  │                       enlil-hal                          │   │\n│  │        HypervisorBackend trait (VMX / SVM / EL2 / H)     │   │\n│  └──────────────────────────┬───────────────────────────────┘  │\n│                             │                                   │\n│  ┌──────────┐         ┌─────┴─────┐         ┌───────────────┐   │\n│  │enlil-boot│         │enlil-setup│         │   enlil-std   │   │\n│  │UEFI stub │         │ setup TUI │         │  std facade   │   │\n│  └──────────┘         └───────────┘         └───────────────┘   │\n├─────────────────────────────────────────────────────────────────┤\n│                     Rust std / core / alloc                     │\n│            (x86_64-unknown-enlil on bare metal, or              │\n│              x86_64-unknown-linux-gnu when hosted)              │\n├─────────────────────────────────────────────────────────────────┤\n│                         enlil-platform                          │\n│   Memory/Alloc · Threading · Sync · Async · Time · I/O          │\n│   ┌─────────────────────────┐  ┌────────────────────────────┐   │\n│   │   platform-linux        │  │   platform-baremetal       │   │\n│   │   (delegates to std)    │  │   (raw HW implementations) │   │\n│   └─────────────────────────┘  └────────────────────────────┘   │\n├─────────────────────────────────────────────────────────────────┤\n│           Hardware  ·  or  ·  Linux + KVM host                  │\n│      x86_64 (VMX/EPT)  ·  ARM (EL2)  ·  RISC-V (H-ext)          │\n└─────────────────────────────────────────────────────────────────┘

The second pillar is the HAL: enlil-hal defines a single architecture-neutral\nHypervisorBackend trait, so the rest of the system never touches VMCS fields, EPT entries,\nor KVM ioctls directly. Adding ARM EL2 or native VMX is "just" a new impl HypervisorBackend\nbehind a #[cfg(target_arch = "…")] gate — the core, device, config, and management crates\ndon't change.

\n
pub trait HypervisorBackend: Send + Sync {\n    type VCpu: Send;\n    type PageTable: Send;\n    type InterruptController: Send;\n\n    fn create_vcpu(&self, config: &VCpuConfig) -> HalResult<Self::VCpu>;\n    fn run_vcpu(&self, vcpu: &mut Self::VCpu) -> HalResult<VmExit>;\n    fn handle_exit(&self, vcpu: &mut Self::VCpu, exit: &VmExit) -> HalResult<bool>;\n    fn map_guest_memory(&self, page_table: &mut Self::PageTable, guest_addr: u64, host_addr: u64, size: u64, writable: bool) -> HalResult<()>;\n    fn inject_interrupt(&self, vcpu: &mut Self::VCpu, controller: &Self::InterruptController, irq: u32) -> HalResult<()>;\n}

Crate map

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
CrateRole
enlil-platformThe foundation. std-equivalent APIs (memory/alloc, threading + scheduler, sync, async runtime, time, I/O) for both hosted (platform-linux, delegates to std) and bare-metal (platform-baremetal, raw hardware) targets. Under platform-baremetal the crate is #![no_std] + alloc, so memory, sync, threading, and time cross-compile for x86_64-unknown-enlil / x86_64-unknown-uefi and are linked + boot-driven by enlil-boot (io/async_rt still hosted-only pending a no_std IoError / reactor).
enlil-stdA std-shaped facade (collections, future, io, sync, thread, time) over enlil-platform, with integration tests — proves the platform layer can carry ordinary Rust on the custom target.
enlil-halThe architecture-neutral HypervisorBackend trait — the only place ISA-specific virtualization details are meant to appear.
enlil-coreThe hypervisor core: VM lifecycle, vCPU management, memory partitioning, EPT, CPUID/SMBIOS/ACPI handling, serial console, timing stealth, vTPM. Carries the RustVMM stack (kvm-ioctls, vm-memory, vm-superio, linux-loader, …) gated to Linux for the KVM-backed dev path.
enlil-devicesThe virtual device library: VirtIO net/block, full ACPI table synthesis, SMBIOS, interrupt controllers (LAPIC/IOAPIC/MSI), timers (PIT/HPET/TSC/paravirt), PS/2, HDA audio, qcow2/raw storage, a virtual xHCI USB stack with routing, an inter-guest bridge (clipboard, drag-and-drop, shared FS, notifications, URL/protocol-handler routing), a display compositor (Enlil Zones, incl. per-monitor multi-monitor tiling), and anti-detection stealth modules (CPUID, timing, LBR, PMC).
enlil-configTOML guest definitions and validation (no overlapping CPU sets, memory, or device assignments).
enlil-mgmtThe management console — clap CLI + ratatui/crossterm TUI for live control (list/start/stop/attach/status).
enlil-setupFirst-run setup wizard: detect hardware, define guests, assign resources, write config.toml.
enlil-bootUEFI boot payload + early kernel for the bare-metal target (Phase 6). A real efi_main collects the boot handoff and calls ExitBootServices(), then kernel_entry brings the machine up under enlil's own control: memory-map walk, switching heap, IDT + int3 self-test, own GDT+TSS with three IST stacks (#DF/#PF/#GP self-tested), spinlock, x2APIC + LAPIC timer (one-shot and TSC-deadline, incl. ns-precise deadline arming), PIT-calibrated TSC with a monotonic ns clock + busy-sleep, per-CPU GS-base TLS, its own identity page tables (with 2 MiB→4 KiB refinement) and a kernel-owned guarded stack it runs the rest of bring-up on, SMP AP bring-up (firmware-discovered low-memory trampoline + INIT-SIPI-SIPI), ACPI/PCI hardware discovery (MADT + APIC-ID inventory / MCFG / DMAR remapping units + device scopes + legacy & ECAM PCI, with memory-BAR read + write-probe sizing), an 8×8-font GOP text console, and AMD SVM enable + VM_HSAVE_PA then runs guests through a real #VMEXIT dispatch loop — each guest in its own isolated RAM at GPA 0 (non-identity NPT), a guest-GPR save/restore shell + VMSAVE/VMLOAD around VMRUN, per-exit emulation (CPUID stealth, port I/O, RDMSR spoof + WRMSR shadow, NPF demand-paging, native compute loop, VMMCALL hypercall), EVENTINJ interrupt injection, a full 64-bit long-mode guest, guest-exception interception (#UD trapped + re-injected to the guest's IDT), a full interrupt round-trip (inject → handle → IRET → resume), timer-driven preemption (a host LAPIC timer preempting a spinning guest through the INTR intercept, then that tick delivered into the guest and handled by its own IDT), and NPT write-protection (a guest store to a read-only page trapped + granted — the dirty-tracking / copy-on-write primitive). QEMU+OVMF boot-proven nightly.
\n

The intended layering is enlil-mgmt/enlil-setup → enlil-config/enlil-core →\nenlil-devices → enlil-hal → enlil-platform, with enlil-boot as the bare-metal entry\npoint over enlil-platform + enlil-core. Some of those edges are still being wired through\nthe HAL (today enlil-core is the KVM-backed VMM and enlil-devices is a standalone library).

\n
\n

Building

\n

Enlil uses a pinned Rust nightly toolchain (see rust-toolchain.toml)\nbecause the bare-metal path needs -Z build-std and some asm/no_std features.

\n
# Hosted development build (Linux/KVM is the default backend)\ncargo build\ncargo test            # workspace unit + integration tests\ncargo clippy\ncargo fmt --check\n\n# Bare-metal target — recompiles std against the Enlil platform layer\ncargo build --target x86_64-unknown-enlil.json -Z build-std=core,alloc,std
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
TargetPlatform featureHypervisor backendUse case
x86_64-unknown-linux-gnuplatform-linuxKVM (via cfg(target_os = "linux") deps)Development (Phases 0–5)
x86_64-unknown-enlilplatform-baremetalNative VMX (bare-metal)Production (Phase 6+)
\n

The platform is selected by feature flag at compile time. The hypervisor backend\n(KVM vs. native VMX) is determined automatically by the target OS — there is no separate\nbackend-* feature flag:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Feature flagEffect
platform-linuxUse the std-delegating platform (default for hosted builds)
platform-baremetalUse the bare-metal platform implementations
\n
\n

The KVM-backed path requires a Linux host with /dev/kvm; live guest-boot tests\nadditionally need nested virtualization. Where that isn't available, those tests are\nskipped rather than faked.

\n
\n
\n

Configuration

\n

Guests are declared in TOML and validated by enlil-config. See\nexamples/two-guests.toml for a complete example.

\n
[[guest]]\nname = \"debian\"\ncpus = [0, 1]\nmemory_mb = 2048\nkernel = \"/boot/vmlinuz\"\ninitrd = \"/boot/initrd.img\"\ncmdline = \"console=ttyS0 root=/dev/vda1\"\n\n[guest.storage]\ntype = \"virtio-blk\"\npath = \"/dev/sda2\"\n\n[guest.network]\ntype = \"virtio-net\"\ntap = \"tap0\"\n\n[[guest]]\nname = \"windows\"\ncpus = [2, 3]\nmemory_mb = 4096\n# …

Validation enforces the invariants that make isolation real: CPU sets must not overlap\nbetween guests, memory regions must not overlap, total allocation must fit physical memory\nminus Enlil's own reservation, and device assignments must be exclusive.

\n
\n

Repository layout

\n
enlil/\n├── Cargo.toml                  # Workspace root (resolver 2, edition 2024)\n├── rust-toolchain.toml         # Pinned nightly\n├── x86_64-unknown-enlil.json   # Custom bare-metal target spec\n├── README.md                   # This file\n├── ROADMAP.md                  # Development roadmap — a [ ]/[x] task checklist of all 11 phases\n├── examples/                   # Sample guest configurations\n├── enlil-platform/             # Platform abstraction (linux + baremetal backends)\n├── enlil-std/                  # std-shaped facade over the platform layer\n├── enlil-hal/                  # HypervisorBackend trait\n├── enlil-core/                 # Hypervisor core (KVM-backed dev path)\n├── enlil-devices/              # Virtual device library\n├── enlil-config/               # TOML guest config + validation\n├── enlil-mgmt/                 # Management CLI/TUI\n├── enlil-setup/                # First-run setup wizard\n└── enlil-boot/                 # UEFI boot payload + early kernel (Phase 6)
\n

Design principles

\n
    \n
  1. Platform-first. enlil-platform exists so every other crate can write normal Rust\nregardless of target. On Linux it's a thin shim over std; on bare metal it provides a\nglobal allocator, cooperative threading, spinlock-based sync, a hardware-counter timer,\nand UART I/O.
    2. \n
  2. Architecture portability via the HAL. The HypervisorBackend trait is the only place\nISA-specific virtualization appears. The core never touches VMCS/EPT/ioctls directly.
    3. \n
  3. Dual-backend strategy. The same crate graph compiles both for a Linux/KVM development\ntarget and for the bare-metal x86_64-unknown-enlil production target.
    4. \n
  4. Custom target triple. x86_64-unknown-enlil.json disables the default std sysroot\nand recompiles std against enlil-platform via -Z build-std.
    5. \n
  5. USB live boot. Enlil is the bootloader — a UEFI application that boots from USB with\nzero modifications to existing drives, coexisting with Windows Boot Manager / GRUB.
    6. \n
  6. cfg-gated architecture code. Every ISA-specific module sits behind\n#[cfg(target_arch = "…")]; enlil-core contains zero architecture-specific imports.
    7. \n
\n
\n

Security model

\n
    \n
  • Guest isolation is non-negotiable. EPT/NPT enforces memory separation in hardware — no\nguest can read another guest's memory or Enlil's own.
    • \n
  • CPU partitioning. Each vCPU is pinned to a physical core; CPU sets must not overlap\n(validated by enlil-config).
    • \n
  • Device isolation. Passthrough devices use the IOMMU (VT-d / SMMU) to restrict DMA to\nthe owning guest's memory.
    • \n
  • Minimal attack surface. In bare-metal mode Enlil runs no OS — it is the OS. No\nsyscalls, no kernel modules, no userspace; the only ring-0/EL2 code is Enlil itself.
    • \n
  • Rust safety. unsafe is minimized and kept to small, audited blocks; it is concentrated in enlil-platform and enlil-hal, with some unsafe in enlil-core/enlil-devices for FFI and CPU instructions.
    • \n
\n

The roadmap goes further with confidential-VM support (AMD SEV-SNP / Intel TDX) and optional\nZK-proof attestation and cross-guest isolation proofs — see ROADMAP.md (Phases 8.7 / 8.9 / 9.12).

\n
\n

Documentation

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
DocumentWhat's in it
README.md (this file)Project overview — what Enlil is, its architecture, the crate layout, how to build, and where things stand today.
ROADMAP.mdThe development roadmap — a [ ]/[x] task checklist of every phase, from scaffold through to the multi-machine mesh.
Developer book (docs/)Browsable developer docs built with mdbook (pinned 0.4.52): layer stack, crate structure, KVM-to-bare-metal migration plan, dev environment, testing, and the docs/CI build. Build with mdbook build docs.
\n
\n

License

\n

Not yet finalized. The roadmap recommends MIT or Apache-2.0 for maximum ecosystem\ncompatibility with the RustVMM crates Enlil builds on.

\n", + "readmeHtml": "

Enlil

\n
\n

Enlil — the Sumerian god who separated heaven from earth, ruled the space between, and assigned domains to lesser gods.

\n
\n

A portable, bare-metal-capable Type-1 hypervisor written in Rust that turns a single\nx86 desktop into multiple transparent virtual PCs — with granular peripheral routing, GPU\nsharing, and a path toward pooling hardware across many physical machines.

\n
    \n
  • Language: 100% Rust (no_std for the core, std for tooling)
    • \n
  • Foundation: the RustVMM crate ecosystem
    • \n
  • Target hardware: x86-64 desktops with Intel VT-x/VT-d or AMD-V/AMD-Vi (ARM & RISC-V planned)
    • \n
  • Boot model: a UEFI application on a USB stick — non-destructive, coexists with your existing OS
    • \n
  • License: MIT
    • \n
  • Security: 1.0 is unaudited — do not run untrusted guests without reviewing the residual risks. See SECURITY.md.
    • \n
\n
\n

What is Enlil?

\n

Most virtualization runs N guests on 1 host. Enlil's near-term goal is to make a single\nmachine feel like several independent PCs — each guest pinned to its own cores and memory,\neach seeing what looks like real, dedicated hardware — and to do it transparently enough\nthat a guest OS (including Windows, with anti-VM-detection hardening) cannot tell it is\nvirtualized.

\n

The longer-term vision is logical machines over a physical pool: Enlil mediates both\ndirections of every hardware interaction (guest→hardware traps and hardware→guest\ninterrupts) and can route them across a pool of nodes. That turns physical machines into\nstateless hardware providers and each guest into a logical machine defined only by a\nresource manifest and a routing table — hardware disaggregation underneath unmodified\noperating systems. See the Core Model\nbelow for the full picture.

\n

A defining usability constraint shapes the whole project: Enlil must be testable without\nmodifying your system. It boots from a USB stick as a standard UEFI application\n(/EFI/BOOT/BOOTX64.EFI), loads its config and per-guest virtual firmware from that stick,\nand leaves every existing drive and bootloader untouched. Unplug it and reboot to return to\nbare metal instantly.

\n
┌─────────────┐  ┌─────────────┐  ┌─────────────┐\n│  Guest OS 1 │  │  Guest OS 2 │  │  Management  │\n│ (Windows)   │  │  (Linux)    │  │   Console    │\n├─────────────┤  ├─────────────┤  ├─────────────┤\n│ Virtual HW  │  │ Virtual HW  │  │  Config API  │\n│ vCPU · vGPU │  │ vCPU · vGPU │  │  USB Route   │\n│ vUSB · vNIC │  │ vUSB · vNIC │  │  GPU Policy  │\n├─────────────┴──┴─────────────┴──┴─────────────┤\n│                  ENLIL CORE                    │\n│   CPU/Mem Sched · GPU Arbiter · USB Router ·   │\n│            Storage / Net Manager               │\n├────────────────────────────────────────────────┤\n│           UEFI Firmware (boot payload)         │\n├────────────────────────────────────────────────┤\n│                Physical Hardware               │\n└────────────────────────────────────────────────┘

Each guest gets its own virtual UEFI (OVMF) and its own bootloader inside the VM — Enlil\nsits below everything and never replaces GRUB or the Windows Boot Manager.

\n
\n

Core Model — Logical Machines over a Physical Pool

\n

The near-term product (transparent virtual PCs on one machine, described in What is Enlil?) is one point on a much larger design. Enlil mediates both directions of every hardware interaction — the guest→hardware path (VM-exits, MMIO/PIO traps, hypercalls, VirtIO kicks) and the hardware→guest path (physical IRQs, DMA completions, input events, captured on the node that owns the device and injected as virtual interrupts on the node running the guest). Because both directions pass through Enlil, the mapping from a guest to the hardware it runs on is arbitrary and may cross machine and network boundaries.

\n

That turns classic virtualization inside out. Instead of N guests on 1 host, Enlil aims for N guests on M hosts, fully composable: each physical machine becomes a stateless hardware node (a resource provider), and each guest becomes a logical machine defined only by a resource manifest plus a routing table. It is hardware disaggregation underneath unmodified operating systems — the ambition behind CXL, RDMA device pools, and LegoOS, but working transparently beneath stock Windows and Linux rather than requiring a custom OS.

\n

Three physical laws bound what such a pool can actually do, and they shape the whole design.

\n

Interconnect latency sets the granularity of sharing

\n

How tightly two nodes can share resources is governed by the latency of the link between them. Local DRAM is ~80 ns; a WAN round-trip is ~20–40 ms — roughly 100,000× slower — so a vCPU can never run against RAM that lives thousands of miles away.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
InterconnectAdded latencyWhat can be pooled transparently
On-board PCIe / CXLsub-µs (~200–400 ns)almost anything, including memory as a NUMA tier and live devices
RDMA, same datacenter~1–5 µsstorage, NIC, GPU-compute, cold-page memory — not a hot vCPU's RAM
WAN, 1000s of miles~20–40 ms RTTcoarse/async only: remote storage, streamed I/O, whole-guest migration, replication
\n

The rule that falls out: a single kernel's tightly-coupled hot CPU+RAM working set stays on one node (or one CXL-NUMA domain). Across distance you move or replicate a guest — you don't share its hot state. Everything else (devices, storage, GPU offload, spare capacity) is poolable with latency-aware placement. To make this practical, every virtual device is built as a front-end/back-end pair behind a pluggable transport (local | CXL | RDMA | network), so remoting a device is a transport swap rather than a rewrite.

\n

ISA sets the boundary of a logical machine

\n

The latency law has a hardware-architecture twin: a single kernel's execution cannot span an ISA boundary. Page-table formats, the interrupt model, atomics, and the memory model are all ISA-fixed, and no machine description a stock OS accepts (ACPI or device tree) can express mixed-ISA SMP. So every logical machine carries a placement key — (ISA, vendor, feature baseline) — frozen at creation; its vCPUs run natively only on matching nodes, and that key defines its live-migration domain. Foreign-ISA nodes can still serve every other lane (device back-ends, memory tiers, fabric work units are ISA-blind).

\n

A guest may optionally run via binary translation, expressed as a per-guest execution mode:

\n
    \n
  • transparent — vCPUs run natively on matching nodes; remote nodes contribute device back-ends only. This is the full-fidelity product.
    • \n
  • hybrid — native vCPUs plus a translated compute device (an opt-in paravirtual accelerator); the native surface stays fully stealthed.
    • \n
  • mesh — all vCPUs translated (JIT); location- and ISA-free, but a compatibility mode only, and explicitly detectable.
    • \n
\n

Each guest manifest declares the floor it will accept (e.g. require = transparent … allow = mesh) and the pool's composition decides what is achievable — Windows-for-ARM on an all-x86 pool admits only in mesh mode, if at all. Modes are switchable at runtime without a guest reboot. Crucially, translation is a compatibility tool, never a transparency one: per-block overhead and timing artifacts defeat the anti-detection guarantees, so the translated layer is always presented to the guest as a device, never as extra CPUs.

\n

Trust domains: your own pool vs. federating with others

\n

Latency and ISA decide what can be pooled and where it runs natively. A third, orthogonal axis — the trust domain — decides how compute must be protected once a pool spans more than one owner.

\n
    \n
  • An "Enlil device" = every node under one owner. Your phone, desktop, and laptop join into a single pool that presents as one logical-machine device. Inside this boundary compute is trusted and shared in the clear — plaintext memory tiers, zero-copy fabric, native vCPU placement — exactly the Core Model above.
    • \n
  • Federating across owners = untrusted compute. Two Enlil devices can share capacity (borrow a friend's idle GPU or cores), but across that boundary a node you don't own must never see plaintext or be trusted for integrity. That is the blind-compute regime: secret-shared / MPC / ZK-verified work units, backed by attestation that the peer runs genuine Enlil.
    • \n
\n

Trust is orthogonal to latency — a node can be near-but-untrusted (a friend's LAN) or far-but-trusted (your own VPS) — so the placement key gains a trust dimension on top of (latency class, ISA, vendor, feature baseline). Each logical machine declares which trust domains may serve it: a sensitive guest pins to trust = owner-only, while fungible, decomposable work may spill to federated-blind capacity. This is what extends N guests on M hosts to M hosts owned by different people.

\n

Transparency and guest-OS feasibility

\n

Transparency is per-guest-family: Enlil presents whatever machine model each OS expects (PC/UEFI, ARM SoC, Apple platform) and defeats that family's specific VM-detection vectors, generalizing "transparent virtual PC" to a transparent virtual machine, platform-shaped per guest. How far that reaches today:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
GuestFeasibility
Linux, *BSD (x86)✅ near-term
Windows (x86)✅ in scope, with VM-detection hardening
Android (x86)✅ near-term
Android (ARM)⏳ needs the ARM backend; Play Integrity is hardware-attested
macOS⚠️ Apple hardware only — license + SMC/board-id, a major transparency lift
iOS❌ research-only — Apple-signed boot chain + Secure Enclave, not transparently virtualizable on non-Apple hardware
\n
\n

Status

\n

Enlil 1.0.0 — the feature set below is implemented, unit-tested, and (for\nthe bare-metal kernel) asserted nightly under QEMU+OVMF. Work continues on the\nlonger-term phases in ROADMAP.md; anything marked ⏳ is planned,\nnot shipped.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AreaState
Platform layer + custom std target (x86_64-unknown-enlil)✅ Implemented & tested (enlil-platform, enlil-std)
HAL portability trait (HypervisorBackend)✅ Defined (enlil-hal)
KVM-backed VMM: single & multi-guest, CPU/memory partitioning✅ Implemented on Linux (enlil-core)
Virtual device layer (VirtIO net/block, interrupts, timers, PS/2, HDA, storage)✅ Implemented (enlil-devices)
Managed vCPU run loop: stealth stack + per-entry platform-timer cadence & LAPIC TSC-deadline firing✅ Implemented on Linux (enlil-core, StealthRunLoop)
Guest-boot orchestrator: config→GuestRuntime→run, bzImage kernel load (setup-header/boot_params/initrd/E820, real·protected·long-mode entry), per-guest CPUID+vTPM+LBR transparency, guest-RAM I/O — driven by the enlil-run binary✅ Implemented on Linux (enlil-core, orchestrator), KVM-verified end to end
Suspend/resume: vCPU state snapshot/restore + automatic ACPI S3 resume — a guest that commits a suspend is resumed in place (FACS waking-vector discovery via the RSDP→XSDT→FADT→FACS walk, re-entry at the waking vector) with no teardown✅ Implemented on Linux (enlil-core, KVM-verified)
USB peripheral routing (virtual xHCI, TRB-level; config-driven [usb.routing] rules assembled into a controller-validated routing registry; Linux sysfs host enumeration + cadence-throttled hot-plug polling; console USB-tab + guest-list view-models/controllers)🚧 Substantially implemented
Windows transparency (ACPI/SMBIOS synthesis incl. S3/S4/S5 sleep states, CPUID identity+topology/timing/LBR stealth, per-guest vTPM with persistent NV state, independent endorsement keys & name-derived seeds; NIC-OUI transparency in config + safe MAC synthesis; anti-detection: IET APERF/MPERF divergence + CPUID hypervisor-tell decoders — with the hypervisor bit, empty 0x40000000 vendor leaf, and preserved Invariant-TSC all KVM-verified from inside a real guest)🚧 Building blocks implemented
Bare-metal UEFI boot (Phase 6)🚧 The bare-metal kernel now boots and brings the machine up under its own control — proven nightly under QEMU+OVMF. The UEFI payload's efi_main collects the ACPI RSDP + GOP framebuffer + final UEFI memory map into a BootHandoff, calls ExitBootServices(), prints its liveness banner to COM1 serial, and transitions into the enlil kernel, which then, with no firmware left: walks the handed-off memory map and reports usable RAM; installs its own heap (a switching global allocator: firmware pool → the kernel's linked-list heap in the largest conventional region) and proves a live allocation; loads its own IDT with exception handlers and self-tests it by taking an int3; enables the local APIC in x2APIC mode; enables AMD SVM (EFER.SVME) and programs the VM_HSAVE_PA host state-save area; runs a guest through a real #VMEXIT dispatch engine — it allocates the guest its own isolated 2 MiB RAM window and maps guest-physical 0 onto it (a non-identity NPT, so the guest sees its RAM at GPA 0 while it lives at a disjoint system-physical base — LOCKED PRINCIPLE 5), writes a real-mode program into it, and VMRUNs in a loop that routes every #VMEXIT through the HAL's arch-neutral VmExit model with a guest-GPR save/restore shell around each entry (VMRUN carries only RAX/RSP/RIP/RFLAGS; the shell swaps the other 14 registers so the guest keeps state across an intercepted-and-resumed instruction). The loop emulates the guest's intercepted instructions: CPUID answered with a stealthed host result — the guest reads CPUID.1:ECX[31] (hypervisor-present) as 0, so it cannot detect enlil in-guest (LOCKED PRINCIPLE 1); port I/O decoded + captured; RDMSR answered with a stealth value and WRMSR shadowed per-guest and read back (the write never reaches host hardware); a nested page fault on an unmapped GPA is demand-mapped (fresh frame + NPT leaf) and the access re-executes; and a native arithmetic loop runs a taken branch to completion with no #VMEXIT — near-native guest execution — before the guest HLTs. The run shell brackets VMRUN with VMSAVE/VMLOAD so the guest's FS/GS/TR/LDTR + SYSENTER state is swapped in and out (a guest using segmentation/syscalls is safe); enlil delivers interrupts into a guest and the guest handles and returns from them: an EVENTINJ-injected interrupt vectors through the guest's real-mode IVT to a handler that does work in guest RAM (which the hypervisor reads back through the NPT window) and IRETs to resume the interrupted code; the same round-trip runs in 64-bit long mode through a real guest GDT + 64-bit IDT interrupt gate + IRETQ (the way a real x86-64 OS handles interrupts); a maskable virtual interrupt posted via the VMCB INT_CONTROL (V_INTR) is correctly held off while the guest masks interrupts (IF=0) and delivered only when it STIs; and enlil forcibly preempts a guest spinning in an infinite loop that never yields by posting that virtual interrupt — pure time-slicing, the mechanism a scheduler quantum uses to reclaim a CPU. That preemption now runs off the real hardware clock: with the VMCB's physical-interrupt (INTR) intercept armed, a host LAPIC timer that fires while a guest is spinning takes an INTR #VMEXIT straight back into enlil — the host reclaims the CPU from a non-cooperative guest on a genuine time base rather than a pre-posted interrupt. enlil then delivers that tick into the guest: it services and EOIs the physical interrupt, re-posts it to the guest as a virtual interrupt, and resumes without advancing RIP, so the guest vectors through its own long-mode IDT to its own handler and returns — the full path a scheduler tick takes from a host hardware clock into a running guest OS. enlil also traps a guest's own exceptions: it arms the VMCB exception-intercept bitmap so a guest #UD (or any vector) takes a #VMEXIT, then re-delivers the fault to the guest's own IDT/IVT (carrying the error code for the vectors that push one) — the guest handles its own fault while enlil observes it. It runs a full interrupt round-trip — an injected interrupt's handler runs, IRETs, and the guest resumes past the injection point (the inject → handle → IRET → resume cycle a virtual timer tick needs). And it write-protects guest memory in the NPT: a guest store to a read-only page takes a present+write nested page fault enlil records and grants (then the store re-executes) — the dirty-page / copy-on-write signal live migration builds on, while reads and instruction fetches from the same page still succeed. enlil services a VMMCALL paravirt hypercall (result returned in the guest RAX); and a full 64-bit long-mode guest runs with paging on, its own CR3 page-table walk resolved through the NPT (the mode a real x86-64 OS boots in). All boot-verified nightly on real nested SVM. The kernel also loads its own GDT + TSS with three Interrupt Stack Table stacks (the #DF, #PF, and #GP handlers each run on their own known-good stack, every IST-switch self-tested), arms the LAPIC timer in both one-shot and TSC-deadline mode (the precise-preemption clock a scheduler quantizes on — it even arms a 2 ms deadline from a nanosecond request and confirms via its own clock that it fired on time), calibrates the TSC against the fixed-rate PIT (~4.2 GHz measured) and turns it into a monotonic nanosecond clock with a busy-sleep (verified by a 5 ms sleep), installs a per-CPU GS-base TLS block, switches to its own identity page tables (off the firmware's, span derived from the memory map + framebuffer) with the low 2 MiB rendered at 4 KiB granularity and virtual address 0 left unmapped as a null-page guard (a stray kernel null-pointer dereference faults into the #PF handler instead of silently touching page 0), and can refine any other 2 MiB region of that live map down to 4 KiB pages, keeping the bulk huge-page-granular while a chosen region is managed page by page. It uses that to run on a stack it owns with a guard page: the whole rest of bring-up executes on kernel-allocated memory instead of the firmware's boot-services stack, with the page below it left unmapped so a stack overflow takes a diagnosable #PF (on its own IST stack) rather than silently corrupting its neighbour — the guard is proven absent by walking the live page tables, not by touching it. It brings up the other CPUs: the BSP finds a firmware-conventional page below 1 MiB for the AP startup trampoline (discovered from the real memory map, not a hardcoded address), installs a 16-bit real-mode trampoline, and wakes each application processor from the MADT inventory with INIT-SIPI-SIPI at the SDM's timings, each AP reporting in on a shared counter before a bounded timeout. It self-tests a spinlock primitive, and draws both a boot indicator and an 8×8-font text banner to the GOP framebuffer (pixel-readback verified). It discovers real hardware from the firmware tables the UEFI stage handed off: ACPI RSDP→XSDT→MADT (enabled-CPU count plus each processor's APIC ID — the SMP AP inventory, booted under -smp 2), MCFG (the PCIe ECAM window), and DMAR/IVRS — for VT-d it parses the DMAR body into each DMA-remapping hardware unit (register block, PCI segment, catch-all flag) and decodes that unit's device scopes (which endpoints, bridges, I/O APICs and HPETs it governs — what can be placed in a per-guest DMA domain), verified nightly against an emulated VT-d IOMMU's real firmware table. It enumerates PCI both via the legacy 0xCF8/0xCFC mechanism (host-bridge identity, device classes, MSI-X capability, and the first memory BAR — base and size via the standard all-ones write-probe, e.g. the real VGA BAR sized at 16 MiB, plus a summary of every memory BAR on the bus — their count and total MMIO footprint, the window the hypervisor must account for in config-space routing and passthrough) and via ECAM MMIO (all buses, extended config space). Each step is asserted on serial by the headless QEMU+OVMF harness (scripts/qemu-boot-test.sh, KVM-accelerated, nested +svm). The HAL carries the decode + region seams both backends need: enlil-hal::vmx (Intel VMX capability MSRs, VMCS field/exit encodings, VMXON/VMCS region init) and enlil-hal::svm (AMD SVM CPUID/VM_CR gates, the VMCB control/save-area layout, #VMEXIT decode), plus enlil-hal::region owning the 4 KiB-aligned VMCS/VMXON/VMCB/host-save pages, enlil-hal::svm's VMCB programming layer (program_minimal_hlt_guest + segment/exit-field accessors) and enlil-hal::npt's nested-page-table identity-map builder — all built for the x86_64-unknown-enlil kernel target. The kernel now links enlil-platform directly (Phase 1.2 — the crate is no_std under platform-baremetal, so its host-agnostic modules cross-compile for the bare-metal target): it drives enlil-platform's work-stealing scheduler (four tasks submitted out of order run Critical→Low on real hardware, their closures heap-boxed through the kernel's own allocator across the crate boundary), and drives that scheduler's quantum from the LAPIC TSC-deadline timer — one dispatch per real hardware tick, each quantum armed as a nanosecond deadline, so the bare-metal scheduler advances on the same clock a preemptive OS quantizes on, builds a MemoryMap from the real firmware map and carves a disjoint DMA / heap / per-guest RAM plan (cross-checked against the kernel's own memory walk — LOCKED PRINCIPLE 5) after marking the already-installed kernel heap as in-use, so the planner can never hand a guest memory the hypervisor is allocating from — and guests now run out of that planned region rather than the kernel heap, so a guest's nested mapping cannot reach the hypervisor's own allocator. It times a busy-sleep with enlil-platform's Instant monotonic clock — all boot-verified, so the boot kernel consumes shared platform primitives instead of re-deriving them. Still landed: discover::build_device_tree assembles a HostDeviceTree from the ACPI tables (CPUs with NUMA nodes via MADT + SRAT, per-node RAM ranges and distances via SRAT/SLIT, a full MCFG-ECAM PCI walk incl. BARs/xHCI/capabilities/SR-IOV, DMAR/IVRS IOMMU kind), plus MemoryMap carve/plan from E820 & UEFI maps and x86-64 identity page-table builders. Next: bringing each started AP from real mode into long mode with its own guarded stack and per-CPU state (multi-vCPU guests follow), programming the discovered IOMMU units' root/context tables for per-guest DMA isolation, and booting a real distro / privileged service VM
GPU sharing, compute fabric, ARM/RISC-V, multi-machine mesh⏳ Planned (Phases 7–11)
\n
\n

Honest caveat: the RustVMM/KVM dependencies in enlil-core are gated to\ntarget_os = "linux", and live guest-boot/integration tests require KVM (nested\nvirtualization). On other hosts the logic-level unit tests still build and run, but a real\nguest boot does not. Wiring enlil-core and enlil-devices fully through enlil-hal is\nin progress. See ROADMAP.md for exactly what is built vs. planned.

\n
\n
\n

Architecture

\n

Enlil is a Cargo workspace of focused crates layered over a custom platform abstraction. The\nkey idea — platform-first — is that every crate above the platform layer writes ordinary\nRust (Vec, HashMap, Arc<Mutex<T>>, async/await), whether the final target is Linux\nor bare metal. The platform crate absorbs all the no_std complexity.

\n
┌─────────────────────────────────────────────────────────────────┐\n│                          Enlil Crates                           │\n│  ┌──────────┐ ┌───────────┐ ┌────────────┐ ┌───────────────┐   │\n│  │enlil-core│ │enlil-mgmt │ │enlil-config│ │ enlil-devices │   │\n│  └────┬─────┘ └─────┬─────┘ └─────┬──────┘ └──────┬────────┘   │\n│       │             │             │               │            │\n│  ┌────┴─────────────┴─────────────┴───────────────┴────────┐   │\n│  │                       enlil-hal                          │   │\n│  │        HypervisorBackend trait (VMX / SVM / EL2 / H)     │   │\n│  └──────────────────────────┬───────────────────────────────┘  │\n│                             │                                   │\n│  ┌──────────┐         ┌─────┴─────┐         ┌───────────────┐   │\n│  │enlil-boot│         │enlil-setup│         │   enlil-std   │   │\n│  │UEFI stub │         │ setup TUI │         │  std facade   │   │\n│  └──────────┘         └───────────┘         └───────────────┘   │\n├─────────────────────────────────────────────────────────────────┤\n│                     Rust std / core / alloc                     │\n│            (x86_64-unknown-enlil on bare metal, or              │\n│              x86_64-unknown-linux-gnu when hosted)              │\n├─────────────────────────────────────────────────────────────────┤\n│                         enlil-platform                          │\n│   Memory/Alloc · Threading · Sync · Async · Time · I/O          │\n│   ┌─────────────────────────┐  ┌────────────────────────────┐   │\n│   │   platform-linux        │  │   platform-baremetal       │   │\n│   │   (delegates to std)    │  │   (raw HW implementations) │   │\n│   └─────────────────────────┘  └────────────────────────────┘   │\n├─────────────────────────────────────────────────────────────────┤\n│           Hardware  ·  or  ·  Linux + KVM host                  │\n│      x86_64 (VMX/EPT)  ·  ARM (EL2)  ·  RISC-V (H-ext)          │\n└─────────────────────────────────────────────────────────────────┘

The second pillar is the HAL: enlil-hal defines a single architecture-neutral\nHypervisorBackend trait, so the rest of the system never touches VMCS fields, EPT entries,\nor KVM ioctls directly. Adding ARM EL2 or native VMX is "just" a new impl HypervisorBackend\nbehind a #[cfg(target_arch = "…")] gate — the core, device, config, and management crates\ndon't change.

\n
pub trait HypervisorBackend: Send + Sync {\n    type VCpu: Send;\n    type PageTable: Send;\n    type InterruptController: Send;\n\n    fn create_vcpu(&self, config: &VCpuConfig) -> HalResult<Self::VCpu>;\n    fn run_vcpu(&self, vcpu: &mut Self::VCpu) -> HalResult<VmExit>;\n    fn handle_exit(&self, vcpu: &mut Self::VCpu, exit: &VmExit) -> HalResult<bool>;\n    fn map_guest_memory(&self, page_table: &mut Self::PageTable, guest_addr: u64, host_addr: u64, size: u64, writable: bool) -> HalResult<()>;\n    fn inject_interrupt(&self, vcpu: &mut Self::VCpu, controller: &Self::InterruptController, irq: u32) -> HalResult<()>;\n}

Crate map

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
CrateRole
enlil-platformThe foundation. std-equivalent APIs (memory/alloc, threading + scheduler, sync, async runtime, time, I/O) for both hosted (platform-linux, delegates to std) and bare-metal (platform-baremetal, raw hardware) targets. Under platform-baremetal the crate is #![no_std] + alloc, so memory, sync, threading, and time cross-compile for x86_64-unknown-enlil / x86_64-unknown-uefi and are linked + boot-driven by enlil-boot (io/async_rt still hosted-only pending a no_std IoError / reactor).
enlil-stdA std-shaped facade (collections, future, io, sync, thread, time) over enlil-platform, with integration tests — proves the platform layer can carry ordinary Rust on the custom target.
enlil-halThe architecture-neutral HypervisorBackend trait — the only place ISA-specific virtualization details are meant to appear.
enlil-coreThe hypervisor core: VM lifecycle, vCPU management, memory partitioning, EPT, CPUID/SMBIOS/ACPI handling, serial console, timing stealth, vTPM. Carries the RustVMM stack (kvm-ioctls, vm-memory, vm-superio, linux-loader, …) gated to Linux for the KVM-backed dev path.
enlil-devicesThe virtual device library: VirtIO net/block, full ACPI table synthesis, SMBIOS, interrupt controllers (LAPIC/IOAPIC/MSI), timers (PIT/HPET/TSC/paravirt), PS/2, HDA audio, qcow2/raw storage, a virtual xHCI USB stack with routing, an inter-guest bridge (clipboard, drag-and-drop, shared FS, notifications, URL/protocol-handler routing), a display compositor (Enlil Zones, incl. per-monitor multi-monitor tiling), and anti-detection stealth modules (CPUID, timing, LBR, PMC).
enlil-configTOML guest definitions and validation (no overlapping CPU sets, memory, or device assignments).
enlil-mgmtThe management console — clap CLI + ratatui/crossterm TUI for live control (list/start/stop/attach/status).
enlil-setupFirst-run setup wizard: detect hardware, define guests, assign resources, write config.toml.
enlil-bootUEFI boot payload + early kernel for the bare-metal target (Phase 6). A real efi_main collects the boot handoff and calls ExitBootServices(), then kernel_entry brings the machine up under enlil's own control: memory-map walk, switching heap, IDT + int3 self-test, own GDT+TSS with three IST stacks (#DF/#PF/#GP self-tested), spinlock, x2APIC + LAPIC timer (one-shot and TSC-deadline, incl. ns-precise deadline arming), PIT-calibrated TSC with a monotonic ns clock + busy-sleep, per-CPU GS-base TLS, its own identity page tables (with 2 MiB→4 KiB refinement) and a kernel-owned guarded stack it runs the rest of bring-up on, SMP AP bring-up (firmware-discovered low-memory trampoline + INIT-SIPI-SIPI), ACPI/PCI hardware discovery (MADT + APIC-ID inventory / MCFG / DMAR remapping units + device scopes + legacy & ECAM PCI, with memory-BAR read + write-probe sizing), an 8×8-font GOP text console, and AMD SVM enable + VM_HSAVE_PA then runs guests through a real #VMEXIT dispatch loop — each guest in its own isolated RAM at GPA 0 (non-identity NPT), a guest-GPR save/restore shell + VMSAVE/VMLOAD around VMRUN, per-exit emulation (CPUID stealth, port I/O, RDMSR spoof + WRMSR shadow, NPF demand-paging, native compute loop, VMMCALL hypercall), EVENTINJ interrupt injection, a full 64-bit long-mode guest, guest-exception interception (#UD trapped + re-injected to the guest's IDT), a full interrupt round-trip (inject → handle → IRET → resume), timer-driven preemption (a host LAPIC timer preempting a spinning guest through the INTR intercept, then that tick delivered into the guest and handled by its own IDT), and NPT write-protection (a guest store to a read-only page trapped + granted — the dirty-tracking / copy-on-write primitive). QEMU+OVMF boot-proven nightly.
\n

The intended layering is enlil-mgmt/enlil-setup → enlil-config/enlil-core →\nenlil-devices → enlil-hal → enlil-platform, with enlil-boot as the bare-metal entry\npoint over enlil-platform + enlil-core. Some of those edges are still being wired through\nthe HAL (today enlil-core is the KVM-backed VMM and enlil-devices is a standalone library).

\n
\n

Building

\n

Enlil uses a pinned Rust nightly toolchain (see rust-toolchain.toml)\nbecause the bare-metal path needs -Z build-std and some asm/no_std features.

\n
# Hosted development build (Linux/KVM is the default backend)\ncargo build\ncargo test            # workspace unit + integration tests\ncargo clippy\ncargo fmt --check\n\n# Bare-metal target — recompiles std against the Enlil platform layer\ncargo build --target x86_64-unknown-enlil.json -Z build-std=core,alloc,std
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
TargetPlatform featureHypervisor backendUse case
x86_64-unknown-linux-gnuplatform-linuxKVM (via cfg(target_os = "linux") deps)Development (Phases 0–5)
x86_64-unknown-enlilplatform-baremetalNative VMX (bare-metal)Production (Phase 6+)
\n

The platform is selected by feature flag at compile time. The hypervisor backend\n(KVM vs. native VMX) is determined automatically by the target OS — there is no separate\nbackend-* feature flag:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Feature flagEffect
platform-linuxUse the std-delegating platform (default for hosted builds)
platform-baremetalUse the bare-metal platform implementations
\n
\n

The KVM-backed path requires a Linux host with /dev/kvm; live guest-boot tests\nadditionally need nested virtualization. Where that isn't available, those tests are\nskipped rather than faked.

\n
\n
\n

Configuration

\n

Guests are declared in TOML and validated by enlil-config. See\nexamples/two-guests.toml for a complete example.

\n
[[guest]]\nname = \"debian\"\ncpus = [0, 1]\nmemory_mb = 2048\nkernel = \"/boot/vmlinuz\"\ninitrd = \"/boot/initrd.img\"\ncmdline = \"console=ttyS0 root=/dev/vda1\"\n\n[guest.storage]\ntype = \"virtio-blk\"\npath = \"/dev/sda2\"\n\n[guest.network]\ntype = \"virtio-net\"\ntap = \"tap0\"\n\n[[guest]]\nname = \"windows\"\ncpus = [2, 3]\nmemory_mb = 4096\n# …

Validation enforces the invariants that make isolation real: CPU sets must not overlap\nbetween guests, memory regions must not overlap, total allocation must fit physical memory\nminus Enlil's own reservation, and device assignments must be exclusive.

\n
\n

Repository layout

\n
enlil/\n├── Cargo.toml                  # Workspace root (resolver 2, edition 2024)\n├── rust-toolchain.toml         # Pinned nightly\n├── x86_64-unknown-enlil.json   # Custom bare-metal target spec\n├── README.md                   # This file\n├── ROADMAP.md                  # Development roadmap — a [ ]/[x] task checklist of all 11 phases\n├── examples/                   # Sample guest configurations\n├── enlil-platform/             # Platform abstraction (linux + baremetal backends)\n├── enlil-std/                  # std-shaped facade over the platform layer\n├── enlil-hal/                  # HypervisorBackend trait\n├── enlil-core/                 # Hypervisor core (KVM-backed dev path)\n├── enlil-devices/              # Virtual device library\n├── enlil-config/               # TOML guest config + validation\n├── enlil-mgmt/                 # Management CLI/TUI\n├── enlil-setup/                # First-run setup wizard\n└── enlil-boot/                 # UEFI boot payload + early kernel (Phase 6)
\n

Design principles

\n
    \n
  1. Platform-first. enlil-platform exists so every other crate can write normal Rust\nregardless of target. On Linux it's a thin shim over std; on bare metal it provides a\nglobal allocator, cooperative threading, spinlock-based sync, a hardware-counter timer,\nand UART I/O.
    2. \n
  2. Architecture portability via the HAL. The HypervisorBackend trait is the only place\nISA-specific virtualization appears. The core never touches VMCS/EPT/ioctls directly.
    3. \n
  3. Dual-backend strategy. The same crate graph compiles both for a Linux/KVM development\ntarget and for the bare-metal x86_64-unknown-enlil production target.
    4. \n
  4. Custom target triple. x86_64-unknown-enlil.json disables the default std sysroot\nand recompiles std against enlil-platform via -Z build-std.
    5. \n
  5. USB live boot. Enlil is the bootloader — a UEFI application that boots from USB with\nzero modifications to existing drives, coexisting with Windows Boot Manager / GRUB.
    6. \n
  6. cfg-gated architecture code. Every ISA-specific module sits behind\n#[cfg(target_arch = "…")]; enlil-core contains zero architecture-specific imports.
    7. \n
\n
\n

Security model

\n
    \n
  • Guest isolation is non-negotiable. EPT/NPT enforces memory separation in hardware — no\nguest can read another guest's memory or Enlil's own.
    • \n
  • CPU partitioning. Each vCPU is pinned to a physical core; CPU sets must not overlap\n(validated by enlil-config).
    • \n
  • Device isolation. Passthrough devices use the IOMMU (VT-d / SMMU) to restrict DMA to\nthe owning guest's memory.
    • \n
  • Minimal attack surface. In bare-metal mode Enlil runs no OS — it is the OS. No\nsyscalls, no kernel modules, no userspace; the only ring-0/EL2 code is Enlil itself.
    • \n
  • Rust safety. unsafe is minimized and kept to small, audited blocks; it is concentrated in enlil-platform and enlil-hal, with some unsafe in enlil-core/enlil-devices for FFI and CPU instructions.
    • \n
\n

The roadmap goes further with confidential-VM support (AMD SEV-SNP / Intel TDX) and optional\nZK-proof attestation and cross-guest isolation proofs — see ROADMAP.md (Phases 8.7 / 8.9 / 9.12).

\n
\n

Documentation

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
DocumentWhat's in it
README.md (this file)Project overview — what Enlil is, its architecture, the crate layout, how to build, and where things stand today.
ROADMAP.mdThe development roadmap — a [ ]/[x] task checklist of every phase, from scaffold through to the multi-machine mesh.
Developer book (docs/)Browsable developer docs built with mdbook (pinned 0.4.52): layer stack, crate structure, KVM-to-bare-metal migration plan, dev environment, testing, and the docs/CI build. Build with mdbook build docs.
\n
\n

License

\n

Not yet finalized. The roadmap recommends MIT or Apache-2.0 for maximum ecosystem\ncompatibility with the RustVMM crates Enlil builds on.

\n", "releases": [ { "tag": "v0.1.0-beta.3", @@ -384,7 +384,7 @@ }, "weftdb": { "repo": "weftdb", - "fetchedAt": "2026-10-13T05:41:34.359Z", + "fetchedAt": "2026-10-14T04:35:14.384Z", "description": null, "htmlUrl": "https://github.com/basic-automation/weftdb", "homepage": null, @@ -392,13 +392,13 @@ "topics": [], "stars": 0, "forks": 0, - "openIssues": 2, + "openIssues": 1, "license": "Apache-2.0", "defaultBranch": "main", "createdAt": "2025-06-15T01:45:58Z", - "pushedAt": "2026-10-11T11:55:21Z", + "pushedAt": "2026-10-13T20:21:24Z", "archived": false, - "readmeHtml": "

WeftDB

\n

The time-series database for data that wasn't sampled on a clean grid.

\n

\"Release\"\n\"CI\"\n\"License:\n\"Rust\"\n\"Platform\"\n\"Status\"

\n

Most time-series databases assume your data arrives on a regular cadence. Real data\noften doesn't. Sensors drop out. Markets trade in bursts. Devices batch-upload after\nhours offline. When you ask those databases for a value between two samples, you get\na gap, a NULL, or a 0 — and the work of reconstructing the signal lands in your\napplication code.

\n

WeftDB puts that reconstruction in the database. Ask for any resolution you like\nand it returns a continuous series, interpolated with the spline method you choose,\nwhere every point is labelled raw, interpolated, or extrapolated — so a\nsynthetic value is never silently mistaken for an observed one.

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/interpolate \\\n  -H 'content-type: application/json' \\\n  -d '{\"spline\":\"linear\",\"resolution\":\"seconds\",\n       \"points\":[{\"timestamp\":\"1970-01-01T00:00:00Z\",\"value\":0.0},\n                 {\"timestamp\":\"1970-01-01T00:01:00Z\",\"value\":60.0}]}'

Two samples a minute apart come back as 61 labelled, per-second points — the two\nobserved endpoints marked raw, the 59 between them interpolated.

\n
\n

Contents

\n

Start here · Why WeftDB · Quickstart · Is WeftDB right for you? · Project status

\n

Use it · What you get · HTTP API · Configuration · Rust library · Terminal UI

\n

Understand it · Performance · How it works · Benchmarking · Development

\n
\n

Why WeftDB

\n

Interpolation is a first-class query, not post-processing. Resampling, gap\nfilling and upsampling run next to the data, over typed columnar segments, on one CPU\nthread, the rayon pool or — once calibrated — the GPU, chosen by grid size, not in a\nclient loop pulling raw rows across the wire.

\n

Precision is declared, never silently lost. Values are logically\nBigDecimal. Each aspect declares a physical encoding\n(F64, F32, ScaledI64, ScaledI128, Decimal128, BigDecimalText) and an\nerror bound. A value the encoding cannot represent within that bound is rejected,\nnot quietly rounded. If you are storing money or calibrated instrument readings, this\nis the difference between a database you can audit and one you can't.

\n

Synthetic data is always marked as synthetic. Every reconstructed point carries\nits provenance. Downstream code can filter, weight or refuse interpolated values on\npurpose instead of by accident.

\n

Claims come with benchmarks. Every performance number in this document links to\nthe harness that produced it, and the harness ships in the repo so you can re-run it\non your hardware. Where WeftDB doesn't win, this README says so.

\n
\n

Quickstart

\n

Prerequisites: Rust 1.95 or newer on stable. A GPU with a wgpu backend\n(Vulkan / Metal / DX12) is optional — without one the engine runs on the CPU\n(one thread or the rayon pool) automatically.

\n
cargo build --release

Prefer not to build from source? Each\nGitHub release, from v0.1.0 on,\ncarries pre-built weft-server, weft-tui and weft-bench archives for Linux (x86_64\nand aarch64), macOS (x86_64 and Apple silicon) and Windows (x86_64), and a SHA256SUMS\nfile to check them with sha256sum -c SHA256SUMS. The Linux binaries are built on\nUbuntu 24.04: weft-server and weft-bench need glibc 2.34 or newer, and weft-tui\nneeds glibc 2.39 or newer. The macOS and Windows binaries are not signed, so Gatekeeper\nand SmartScreen will warn before running them. Read the release's known limitations\n(v0.1.0's are in its\nchangelog)\nbefore you run it.

\n

1. Start the server

\n
WEFT_SEGMENT_STORE_ROOT=./store cargo run --release -p weft-server\n# weft-server v0.1.0 listening on http://127.0.0.1:8080

WEFT_SEGMENT_STORE_ROOT is what enables the persistent /storage endpoints;\nwithout it the compute endpoints still work statelessly.

\n

2. Interpolate without storing anything

\n

The compute endpoints take points in the request, so you can evaluate WeftDB\nbefore committing any data to it:

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/interpolate \\\n  -H 'content-type: application/json' \\\n  -d '{\"spline\":\"cubic\",\"resolution\":\"seconds\",\n       \"points\":[{\"timestamp\":\"1970-01-01T00:00:00Z\",\"value\":0.0},\n                 {\"timestamp\":\"1970-01-01T00:00:30Z\",\"value\":30.0},\n                 {\"timestamp\":\"1970-01-01T00:01:00Z\",\"value\":60.0}]}'

Already have InfluxDB Line Protocol? Point it at the ILP sibling instead:

\n
printf 'cpu,host=a load=0 1000000000\\ncpu,host=a load=60 1000000060\\n' | \\\n  curl -s -X POST \\\n    'http://127.0.0.1:8080/api/v1/interpolate/ilp?field=load&precision=s&spline=linear&resolution=seconds' \\\n    -H 'content-type: text/plain' --data-binary @-

3. Store a series and query it back

\n

Declare the aspect's schema once — this is where you choose precision:

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/storage/aspects \\\n  -H 'content-type: application/json' \\\n  -d '{\"name\":\"price\",\"physical_type\":\"scaled_i64\",\"scale\":2,\n       \"value_tolerance\":\"0\",\"timestamp_unit\":\"millis\"}'

scale: 2 with value_tolerance: "0" means exactly two decimal places, no rounding\npermitted — a third decimal is rejected at ingest rather than silently truncated.

\n
# ingest\ncurl -s -X POST http://127.0.0.1:8080/api/v1/storage/price/points \\\n  -H 'content-type: application/json' \\\n  -d '{\"points\":[{\"timestamp\":1000,\"value\":\"1.25\"},\n                 {\"timestamp\":2000,\"value\":\"2.50\"},\n                 {\"timestamp\":3000,\"value\":\"3.75\"}]}'\n\n# read the range back (start/end are required, in the aspect's declared unit)\ncurl -s 'http://127.0.0.1:8080/api/v1/storage/price/points?start=0&end=9999&limit=10'\n\n# read one instant\ncurl -s 'http://127.0.0.1:8080/api/v1/storage/price/at?t=2000'\n\n# bucket it\ncurl -s 'http://127.0.0.1:8080/api/v1/storage/price/downsample?resolution=seconds&agg=avg,max,p95'

Every read surface also speaks CSV, Arrow IPC and Parquet — append .csv,\n.arrow or .parquet, or use the …/range.parquet variants, and pipe straight into\npandas, Polars or DuckDB.

\n
\n

Is WeftDB right for you?

\n

Being honest about this saves you an evaluation.

\n

A good fit

\n
    \n
  • Irregular or gappy series where you need values between the samples —\nsensor networks with dropouts, market data, batch-uploaded device telemetry.
    • \n
  • Resampling and gap filling as a hot path, run often enough that doing it in\napplication code is costing you real latency or CPU.
    • \n
  • Values where precision is not negotiable — prices, billing quantities,\ncalibrated scientific readings — and a silent f64 round-trip is unacceptable.
    • \n
  • Workloads that care which points are real, because a synthetic value would\notherwise contaminate an alert, a model, or an invoice.
    • \n
  • Point lookups and narrow windows over large columnar segments, which WeftDB\noptimises specifically (see Performance).
    • \n
\n

A poor fit — today

\n
    \n
  • You need a mature, hardened production database. WeftDB has no TLS, no\nauthentication, and no RBAC yet. Do not expose it to an untrusted network.
    • \n
  • Generic OLAP over wide tables. WeftDB is not competing with ClickHouse or\nDuckDB on broad analytical scans, and doesn't claim to.
    • \n
  • Regular, evenly-sampled data with no gaps. If interpolation isn't your\nproblem, WeftDB's core advantage doesn't apply and a conventional TSDB is the\nsimpler choice.
    • \n
  • Per-measurement tags / labels. Not implemented yet, so tag-based filtering\nand multi-dimensional grouping aren't available.
    • \n
  • High-cardinality dimensional slicing in the Prometheus sense — that's not\nthe shape of WeftDB's data model.
    • \n
  • Managed hosting. There is none; you run it yourself.
    • \n
\n
\n

Project status

\n

Pre-beta. Expect sharp edges. WeftDB is an engineering-grade codebase — over\n1,000 tests pass, the storage format is CRC-checksummed and versioned, and the\nbenchmarks are real and reproducible — but it has not been hardened for production\ndeployment.

\n

v0.1.0 is the pre-beta baseline release: pre-built binaries on the\nGitHub release, and no\ncrates on crates.io. Later releases test their upgrade, rollback and compatibility\nagainst it. Its\nknown limitations\ninclude no\nauthentication or TLS, incomplete crash consistency, and a store layout that the next\nrelease upgrades one way.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AreaState
Storage engine, interpolation, HTTP API, Arrow/ParquetWorking, tested, benchmarked
Backup & restore (control plane)Working, with verification and a rehearsal drill
Observability — Prometheus metrics, OpenTelemetry tracesWorking
Crash consistency — power-loss durability, crash-safe maintenanceIn progress (design)
Security — TLS, auth, RBAC, audit logsNot started
Pre-built binaries (Linux, macOS, Windows)Released: v0.1.0, on GitHub
Library crates on crates.ioNot published yet
Packaging — Docker, HelmNot started
Client SDKs (Python, TypeScript)Not started
Per-measurement tagsNot started
\n

The full work queue lives in ROADMAP.md. It is the single source of\ntruth for status and priorities, and it records negative results alongside wins.

\n
\n

What you get

\n

Query

\n
    \n
  • Interpolation on read — reconstruct any series onto any grid from nanoseconds\nto years, with Linear / Quadratic / Cubic / Polynomial splines. The engine picks\nsingle-threaded CPU, parallel (Rayon) or — once calibrated — GPU by grid size, and\ndegrades the method gracefully (Cubic → Quadratic → Linear) when data is too sparse\nto support it.
    • \n
  • Provenance on every point — raw, interpolated or extrapolated.
    • \n
  • Downsampling into epoch-aligned buckets: min/max/avg/sum/first/last,\nexact nearest-rank p50/p90/p95/p99, three time-weighted averages (LOCF\nfor change-only sensors, trapezoidal for continuous signals, and an LOCF variant\nthat carries the last sample to the bucket's end), plus bounded-error\nsketch_p* percentiles that are mergeable and cost a fixed amount of memory per\nbucket however many samples land in it.
    • \n
  • Point and range reads that skip work: a point lookup on a regular column is\nresolved in closed form without materialising timestamps at all.
    • \n
\n

Store

\n
    \n
  • Typed columnar segments (.weftseg) — written once, never rewritten: maintenance\n(reconcile, split, overlap merge, squash, compaction) writes its output to new frames,\nfsyncs them, and swaps them for its inputs in one control-plane transaction, so a\ncrash leaves either the inputs or the outputs, and a replaced frame is deleted only\nonce no read can still be using it. CRC-checksummed, page-indexed, with per-segment\nand per-page min/max statistics for data skipping and a null/quality column.
    • \n
  • Compression that adapts per column — timestamps pick among delta-of-delta,\nRLE, Gorilla, fixed and per-block bit-packing; values pick among varint,\nbit-packing, per-block adaptive packing, frame-of-reference and an opt-in delta\ncascade. The codec each column actually used is reported, and bytes/point is the\nrealized figure, not an estimate.
    • \n
  • Late and out-of-order data — detected, counted, and reconciled within a segment or\nacross segments with last-writer-wins semantics, on demand or by background sweep.
    • \n
  • Backup and restore of the control plane, with two verification modes (one safe\nunder concurrent writes) and a non-destructive restore drill so you can answer\n"is my backup actually restorable?" without risking the live store.
    • \n
\n

Integrate

\n
    \n
  • HTTP API with JSON, CSV, Arrow IPC, Parquet and InfluxDB Line Protocol in and\nout — so TSBS/Influx/QuestDB-shaped writers work unchanged, and results land\ndirectly in Arrow-native tooling.
    • \n
  • Prometheus /metrics with latency histograms, plus a live p50/p95/p99 profile\nendpoint and OpenTelemetry trace export.
    • \n
  • Rust libraries — use the engine, the storage layer or the reductions directly,\nwithout the server.
    • \n
  • Terminal UI for importing CSV, browsing and plotting aspects, and running\ncompression interactively.
    • \n
\n
\n

Performance

\n

Every number below comes from a benchmark in this repo, named so you can re-run it. The\nexception is any figure measured on the real BTC corpus: that input is a local file the\nrepository does not ship (see Ingest), so those figures cannot be reproduced from a\nclone. All were measured on one developer workstation — treat them as shape, not as a\nspecification for your hardware.

\n

Reads WeftDB optimises specifically

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OperationWeftDBNaive full decodeGain
Point lookup, 100k-row segment56 µs2.60 ms~46×
Point lookup, paged frame33 µs279 µs~8.5×
Batch of 64 instants (one pass vs 64 single lookups)67 µs3.57 ms~54×
100-row window over 100k rows58 µs2.73 ms~47×
\n

Source: weft-physical-type/benches/pointread.rs,\nre-measured 2026-10-08 after the word-wise bit reads and the slicing-by-16 frame CRC.\nThe wins come from skipping the value column entirely until a specific row is needed,\nand from resolving regular timestamp columns in closed form.

\n

Ingest

\n

Sealing 1M rows of a real 1-minute financial series into a columnar segment:\n4.57 s — 218,963 rows/sec at 4.22 bytes/point.

\n

That corpus (database/datasets/btc_1min.csv, a BTC/USD one-minute series) is a local,\ngitignored file, not distributed with this repository and not required to build or test —\nthe suite skips the tests that use it when it is absent, and so do the benches that load it by\ndefault. Every "real BTC" figure in this README comes from it, so none of them can be reproduced\nfrom a clone. Whether those numbers are cleared for publication or retired is an open item with\ncounsel before 1.0 (1.0 plan, Appendix B).\nThe ingest figure is quoted anyway because a synthetic generator materially flatters the\nresult: on the same workload the generated corpus reports 1.71 bytes/point against the real\n4.22, roughly 2.5× too optimistic. Storage numbers measured on generated data are not reported\nas results.

\n

Reductions

\n

At 500k points, hour buckets, correctness-gated\n(weft-bench --downsample; re-measured 2026-10-08, full table under\nCompute endpoints): avg runs at 2.08M points/sec;\ntime-weighted average costs about 1.39× that. The approximate sketch_p99 is\n~2.6× faster than exact p99 (327 ms vs 852 ms) and stays within its 1% bound,\nwhile using bounded memory per bucket.

\n

Where it doesn't pay off

\n

A deliberately-kept honest result: the bit-sliced (transposed) value codec once\nmeasured ~5.7× faster than the linear bit-unpack at the kernel level, but only because\nthe linear decoder tested one bit at a time. Once that decoder moved to word-wise reads,\nthe linear layout unpacked faster (1.09 vs 1.95 ms per 1 Mi values,\nbenches/bitunpack.rs). End to end, the\nbit-sliced layout costs +0.08% bytes and reads no faster than the linear codec\n(benches/transposed_read.rs), so it\nships off by default (and is now behind the bitsliced-codec build feature, pending\npatent review). A kernel speedup measured against a weak baseline is not a result, and\nthis README would rather say so than quote the 5.7×.

\n

Exact decimals are not free yet either. Hourly avg over 1M real BTC closes (the local corpus\ndescribed under Ingest, not in the repository), all three paths proven to produce the\nsame buckets:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
pathtimevs f64
f64 loop (lossy baseline)2.14 ms1×
weft_reduce::reduce_scaled (exact, on the stored ScaledI64 mantissas)13.52 ms~6.3×
weft_reduce::reduce (exact, per-sample BigDecimal)60.04 ms~28×
\n

QuestDB documents ~2× for its DECIMAL, so\nthe precision wedge still costs more here. Before this work the shipped path measured 100.65 ms\n(~43×). Two changes closed most of the gap: integer\naccumulation, and computing each bucket's avg by integer long division that reproduces\nbigdecimal's quotient digit for digit. Per sealed 1M-row segment (decode + six reductions), the\nstored-range downsample went from 127.6 ms to 59.7 ms. The server's GET /api/v1/storage/{aspect}/downsample now takes the integer path for ScaledI64 segments when every\nrequested reduction is streaming or a sketch_p*, and falls back otherwise\n(weft-reduce/benches/decimal_tax.rs, load average ~11).

\n
\n

HTTP API

\n

weft-server is the axum HTTP surface — Weft-Bench, TSBS-style harnesses, Grafana\nand SDKs all drive the engine through it. It carries no vendor-specific dependencies.

\n

See the Quickstart to start it.

\n

Bind address defaults to 127.0.0.1:8080 (WEFT_SERVER_ADDR overrides). Setting\nWEFT_SEGMENT_STORE_ROOT opens a Storage v2 segment store and enables the\n/storage endpoints (without it they answer 503, and GET /ready reports the\nsegment_store dependency). A store root belongs to one server at a time: the server\nholds the root's LOCK file while it runs, and a second server pointed at the same\nroot exits at startup with an error naming the first one's pid and host.

\n

The root's STORE_FORMAT file records the store's layout and the oldest layouts a\nWeftDB must know to read and to write it (plain JSON; see weftdb::StoreFormat), and\nsegment_index.db's store_meta table keeps a copy. The server reads the marker before\nit opens any database (a root without one, the copy, before it writes anything), and\nrefuses a store a newer WeftDB wrote (IncompatibleLayout) without changing anything\nbut LOCK and LOCK.holder. A store written before the marker existed is layout 1 and\nis upgraded in place on first open, through the registered migrations\n(store_migrations in segment_index.db lists those applied); its frames are not\nrewritten. A layout-1 store whose index records a frame outside segments/ (possible\nonly through the aspect-name traversal of earlier builds) is refused with\nUnsafeLegacyPath, naming the aspects, before any migration applies or a marker is\nwritten, and nothing is moved: drop or re-seal those aspects with the build that wrote\nthem first.

\n

While bound to a loopback address, the server only answers requests addressed to a\nloopback host: Host must be localhost, an address in 127.0.0.0/8 or [::1]\n(anything else is a 421), and a state-changing request whose Origin is not a\nloopback origin is a 403. That keeps web pages from non-loopback origins, open in a\nlocal browser, from driving the server; a page served by another local web server (any\nloopback origin, on any port) is still trusted until authentication lands. Clients that\nsend no Origin, like curl, are unaffected. /health and /ready follow the same\nrule, so a health probe must send a loopback Host such as localhost or 127.0.0.1,\nas every HTTP/1.1 client does; an HTTP/1.0 probe that sends no Host gets a 421, so\nconfigure it to send one or set WEFT_ALLOW_ANY_HOST=1. Behind a local reverse\nproxy that forwards a different Host, set WEFT_ALLOW_ANY_HOST=1; a proxy that keeps\nHost: 127.0.0.1 but forwards the Origin of a browser UI served from a non-loopback\norigin needs it too, or that UI's state-changing requests get a 403. A non-loopback\nbind is not guarded: there is no authentication yet, so keep such a server off\nuntrusted networks.

\n

Service endpoints

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Method & pathPurpose
GET /healthLiveness.
GET /readyReadiness, including the segment-store dependency check. With a store, poisoned / restart_required report that a segment-index COMMIT failed in a way that may still have committed, so the store refuses writes until the server restarts (reads keep working, and ready stays true); poison_reason says which transaction and error.
GET /metricsPrometheus text exposition — request/error/output counters, ingest counters (weft_ingest_*), and latency histograms for the compute and storage-ingest paths.
GET /debug/profile/currentLive p50/p95/p99 latency snapshot per instrumented path.
\n

Compute endpoints

\n

The flagship interpolation-on-read path and its reduction counterpart. Each\naccepts JSON points or an InfluxDB-Line-Protocol body, and each returns JSON,\nCSV, Arrow IPC, or Parquet:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
EndpointPurpose
POST /api/v1/interpolateReconstruct an irregular series onto a regular grid (spline = linear | quadratic | cubic | polynomial; resolution = nanoseconds..years). Every output point carries a kind — raw / interpolated / extrapolated. A grid of more than 10,000,000 points (MAX_INTERPOLATE_OUTPUT_POINTS, or WEFT_MAX_INTERPOLATE_POINTS) is refused with 400 before anything is computed; split larger ranges across requests. The response's spline is the method that ran: with fewer distinct timestamps than the requested method needs (cubic 4, quadratic 3, polynomial degree + 1) it is the simpler method splimes stepped down to.
POST /api/v1/interpolate/pointEvaluate the reconstructed signal at a single instant, labelled raw/interpolated/extrapolated.
POST /api/v1/downsampleReduce samples into epoch-grid-aligned buckets — min/max/avg/sum/first/last, nearest-rank percentiles p50/p90/p95/p99, the three time-weighted averages (twa, LOCF dwell-weighting, for change-only sensors; twa_linear, trapezoidal Σ½(vᵢ+vᵢ₊₁)Δtᵢ/ΣΔtᵢ, for irregularly-sampled continuous signals; twa_bucket_end, LOCF that additionally carries the bucket's last sample to its grid end — twa gives that sample no weight, so a sensor reporting 0 then 100 a minute into an hour bucket reads twa=0 but twa_bucket_end=98.33), and the approximate sketch_p50/p90/p95/p99 (see below); all reductions computed in BigDecimal by the shared weft-reduce crate. Aliases: median→p50, time_weighted_avg/twa_locf→twa, time_weighted_avg_linear→twa_linear, twa_locf_end→twa_bucket_end, sketch_median→sketch_p50. Only non-empty buckets are emitted.
POST /api/v1/{interpolate,downsample}/ilpThe same, fed an ILP text/plain body (the TSBS/InfluxDB/QuestDB wire format); field, precision (ns/us/ms/s), and the compute knobs are query parameters. interpolation= is accepted as an alias for spline= (the canonical spline wins if both are given).
POST /api/v1/{interpolate,downsample}/{csv,arrow,parquet} and …/ilp/{csv,arrow,parquet}The same computations with CSV (text/csv), Arrow IPC stream, or Parquet output — so a harness feeding line protocol pulls results in any of the four formats.
\n

What the reductions cost. Measured on the shipped harness (500k points, 5 reps, hour buckets,\ncorrectness PASS throughout — weft-bench --downsample --ds-points 500000 --ds-aggs <agg>;\nre-measured 2026-10-08 after the avg division change, at load average ~19). These\nuse the generated series, whose values are exact binary expansions of floats (~50 significant\ndigits), so they are a pessimistic bound. On real two-decimal prices (--ds-csv, 1M BTC/USD\none-minute closes from the local corpus described under Ingest, which is not distributed\nwith the repository; hourly avg,p99,twa) the same reduction runs at 4.24M points/sec against\n0.56M points/sec for a generated series of the same size (weft-bench --downsample --ds-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --ds-points 1000000 --ds-bucket h --ds-aggs avg,p99,twa --reps 5):

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
reductionp50throughputnote
avg240.8 ms2,075,477 points/secthe streaming baseline — no bucket materialized
twa334.1 ms1,515,029 points/sec1.39× the streaming cost: dwell-weighting needs the time-ordered samples
twa_bucket_end332.1 ms1,518,520 points/secwithin noise of twa — one extra weight, effectively free
twa_linear407.8 ms1,197,606 points/sec1.22× twa: an extra BigDecimal add + divide per interval for the trapezoidal mean
\n

Exact vs sketch percentiles — which to ask for. The p50/p90/p95/p99 reductions are\nexact nearest-rank: they return an actual observed BigDecimal from the bucket, but they\nmaterialize and sort the whole bucket, and two buckets' results cannot be combined. The\nsketch_p50/p90/p95/p99 reductions instead stream every value into a\nDDSketch — a 1% relative-error bound\n(weft_reduce::SKETCH_ALPHA), bounded memory regardless of bucket size (values fold in as they\narrive; SKETCH_MAX_BINS caps the store absolutely), and an exactly mergeable structure, so a\np99 can be computed over a large or streaming bucket. Measured on the shipped harness (500k points,\n5 reps, correctness PASS): sketch_p99 runs at 1,528,361 points/sec (p50 = 327.30 ms) versus\nexact p99 at 584,469 points/sec (p50 = 852.39 ms) — ~2.6× faster\n(weft-bench --downsample --ds-points 500000 --ds-aggs sketch_p99 vs --ds-aggs p99).

\n

The approximation is declared, never silent — WeftDB's precision principle. The sketch shares the\nexact percentiles' nearest-rank convention (⌈q·n⌉), so sketch_p* and p* name the same\nsample at every bucket size and the sketch is always within the 1% bound of the exact answer —\nthe two are substitutable. (DDSketch's reference rank is ⌊q·(n-1)⌋, which on a small bucket picks\na different sample; WeftDB deliberately does not inherit that.) The exact percentiles remain the\ndefault and cost nothing on a small bucket, so prefer them there; reach for a sketch when a bucket\nis too large to materialize or the result must merge.

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/interpolate \\\n  -H 'content-type: application/json' \\\n  -d '{\n        \"spline\": \"linear\",\n        \"resolution\": \"seconds\",\n        \"points\": [\n          { \"timestamp\": \"1970-01-01T00:00:00Z\", \"value\": 0.0 },\n          { \"timestamp\": \"1970-01-01T00:01:00Z\", \"value\": 60.0 }\n        ]\n      }'
printf 'cpu,host=a load=0 1000000000\\ncpu,host=a load=60 1000000060\\n' | \\\n  curl -s -X POST \\\n    'http://127.0.0.1:8080/api/v1/interpolate/ilp?field=load&precision=s&spline=linear&resolution=seconds' \\\n    -H 'content-type: text/plain' --data-binary @-

Malformed payloads, unknown tokens, fewer than two usable points, or a zero-span\nseries return 400 with an {"error": "..."} body.

\n

Numeric boundary: the compute endpoints' wire values are f64 — the\ndocumented transport boundary (the logical BigDecimal is narrowed only at the\nHTTP edge, and the columnar value columns are honestly typed Float64 — no false\nprecision). The storage endpoints below are lossless.

\n

Storage endpoints (Storage v2 over HTTP)

\n

Catalog management, ingest, and stored-range reads against the .weftseg segment\nstore. The no-silent-downcast guarantee holds end to end: a value unrepresentable\nunder the declared encoding/tolerance is rejected 400.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
EndpointPurpose
POST /api/v1/storage/aspectsDeclare an aspect's schema — physical encoding, value tolerance, timestamp unit. The name also names the aspect's files on disk, so it must be at most 160 bytes with no /, \\, control characters, bidirectional controls, line separators or invisible formatting characters (the zero-width joiner and non-joiner are allowed), leading ., trailing . or space, and must not be a Windows device name (CON, NUL, COM1, …, also with an extension or a : suffix, as in nul:x); a server running on Windows also refuses <, >, :, ", `
GET /api/v1/storage/aspects · …/{aspect}/schema · …/catalogList declared schemas; read one aspect's schema; report the store's (database, subject) scope + registered hierarchy.
POST /api/v1/storage/{aspect}/pointsIngest a JSON batch (dense or nullable; single-block or paged via rows_per_page). Set require_sorted to reject an out-of-order batch (400, naming the first backwards row) instead of sealing it. Every ingest response reports time_sorted (whether the sealed segment stored in monotonic order).
POST /api/v1/storage/{aspect}/{ilp,parquet,csv}Ingest an ILP payload, a Parquet file, or CSV rows into a declared aspect — all sealing through the same schema-enforced path, and all honouring require_sorted (query param) + reporting time_sorted.
GET /api/v1/storage/{aspect}/range · …/value-rangeStream a stored time window / value band as Arrow IPC (application/vnd.apache.arrow.stream).
GET …/range.parquet · …/value-range.parquetThe same windows as Parquet files.
GET …/range.csv · …/value-range.csvThe same windows as CSV (lossless decimal-text values; empty field = null).
GET /api/v1/storage/{aspect}/points · …/value-pointsLossless JSON reads with declarative pagination — offset/limit (+ take and 1-based page aliases), with total/count/offset in the body. For stable forward iteration the body also carries an opaque next_cursor while rows remain; a client re-issues with ?cursor=<token> (supersedes offset/page; malformed → 400) until it is absent.
GET /api/v1/storage/{aspect}/at?t=Single-instant point lookup: the present value at exactly t (lossless decimal text) or a found:false miss. An order-signal-driven read planner resolves each candidate segment with its persisted time_sorted flag — binary search on a sorted segment, linear scan only on an out-of-order one — after pruning the index to the files spanning t.
GET /api/v1/storage/{aspect}/at-multi?t=,,Batch point lookup: a comma-separated list of instants resolved in one pass (points:[{timestamp,value,found}] in query order). The index is pruned once by the batch's whole span and each segment's timestamp column decoded once for the whole batch, so N instants sharing a segment cost one decode, not N.
GET·POST /api/v1/storage/{aspect}/downsample?start=&end=&resolution=&agg=Stored-range downsample — the same reduction set as POST /api/v1/downsample (above), but over the aspect's persisted segments instead of a request-supplied series, so a client aggregates a large stored range without fetching it. Bounded memory: the index is pruned by time and each surviving segment folds into its own mergeable partial reduction, merged once — only one segment's rows are ever resident, so a range far larger than RAM reduces, and with a sketch_p* the per-bucket state is bounded too. Omitted bounds span all stored history. …/downsample.csv · …/downsample.arrow · …/downsample.parquet serve the identical reduction in the compute endpoints' columnar formats. Traced as a downsample.range stage span. The pruned segments are reduced concurrently: measured 4.69× faster than the sequential fold (16 segments over 200k rows, 39.88 ms → 8.50 ms; sketch_p99 49.49 ms → 9.42 ms, 5.25×) — see database/benches/downsample_range.rs. A single pruned segment takes an inline fast path (concurrency there costs more than it buys). Materialized partials (opt-in): with WEFT_SEGMENT_PARTIAL_BASE set, each segment carries a .weftpart partial-reduction sidecar written at seal (a sealed segment is immutable, so its partial never goes stale), and a downsample of the bounded reductions (min/max/avg/sum/first/last + sketch_p*) merges the stored partials instead of decoding the value column — measured 3.2× faster (16 segments / 200k rows, 8.39 ms → 2.60 ms, non-overlapping CIs) and re-keyed to any coarser nesting resolution (minutes→hours→days) with no decode. Rollup tiers (opt-in): WEFT_SEGMENT_PARTIAL_TIERS materializes coarser tiers beside the base (e.g. hours,days), each re-keyed from the tier below, so a coarse query folds the coarsest matching tier instead of the whole fine base — the in-storage analogue of TimescaleDB's hierarchical continuous aggregates. Measured ~1.16× on a DAY query over a MINUTES base (16 segments / 200k rows, 7.30 ms → 6.30 ms, non-overlapping CIs). The win grows with the base-bucket count per segment, and that scaling is now measured rather than asserted: holding rows and segments fixed while widening the sample stride (so only the base buckets per segment change) gives 1.15× at ~208 base buckets → 1.27× at ~3125 → 1.38× at ~12500 (bench_tiered_span). Honest read: the win is real and climbs, but a ~60× increase in base buckets buys only 1.15×→1.38×, so the elided re-key is not the dominant cost and the tiers' extra .weftpart bytes should be weighed against it — see database/benches/downsample_range.rs. A reconcile/split/squash regenerates the sidecar (tiers and all) so the acceleration survives a rewrite.
POST /api/v1/storage/{aspect}/reconcileReconciliation pass (mode in the response). Default (intra-segment): rewrite every out-of-order segment of the aspect into a time-sorted one in place. ?threshold=N gates it on unsorted_segments >= N (QuestDB-style split-count trigger). ?hot_cold=true reconciles cold segments but defers the hot tail until the backlog reaches the threshold. ?overlaps=true instead runs the cross-segment overlap merge (newer-wins) — reconciled is then the number of segments merged away, and ?split_min_bytes=N makes that merge split-not-rewrite (carve off a dominant cold prefix instead of rewriting the whole component). Returns triggered/reconciled/cold_reconciled/hot_reconciled and the post-pass unsorted_segments + overlapping_segments. Maintenance operations on one aspect take turns: while another one holds the aspect (a daemon pass, another request), the request waits up to 30 s, then answers 409 Conflict without touching the aspect; retry.
POST /api/v1/storage/{aspect}/squashSquash the aspect's segments into one (newer-wins), bounding split-path fragmentation. ?max_segments=N gates it (squash only when the count exceeds N). Returns triggered/removed/segment_count. Waits for a busy aspect and answers 409 as reconcile does.
POST /api/v1/storage/{aspect}/compact?target_rows=NSize-targeted compaction — coalesce the aspect's segments toward ~N rows per segment (leaving already-large segments untouched), holding fragmentation near the read-optimal size rather than folding to one (which squash does). Motivated by the downsample_range knee (one giant segment reads slower than several mid-sized ones). target_rows is required (absent → 400). Returns removed/segment_count. The manual counterpart of the WEFT_COMPACT_TARGET_ROWS daemon pass. Waits for a busy aspect and answers 409 as reconcile does.
POST /api/v1/storage/reconcileStore-wide reconciliation sweep across every declared aspect: threshold (default), ?hot_cold=true, or ?overlaps=true (+ ?split_min_bytes=N for split-not-rewrite). Returns mode, aspects_scanned / aspects_reconciled / segments_reconciled (+ cold/hot split), the post-sweep store-wide unsorted_segments + overlapping_segments, and failed: a [{aspect, error}] list of the aspects whose pass failed (empty on a clean sweep). A failing aspect (e.g. a torn or truncated frame) no longer aborts the sweep: every other aspect is still swept and the response is 200, so check failed to tell a partial sweep from a clean one (earlier versions returned 500 at the first failing aspect and left every aspect after it in name order unswept). Only an unreadable aspect list is still a 500. An aspect another maintenance operation holds is waited for (30 s in all, across the sweep); if some are still held then, the others are swept and the answer is 409 Conflict naming the held ones. (The background daemon never waits: it skips a busy aspect until its next tick.) The manual counterpart to the background reconcile daemon (WEFT_RECONCILE_INTERVAL_SECS / WEFT_RECONCILE_THRESHOLD / WEFT_RECONCILE_HOT_COLD / WEFT_RECONCILE_OVERLAPS / WEFT_RECONCILE_SPLIT_MIN_BYTES / WEFT_RECONCILE_MAX_SPLITS).
POST /api/v1/storage/backupOnline control-plane backup (Phase 7.4) — snapshot the store's four control-plane DBs (segment_index/metadata/aspect_catalog/catalog) to fresh files via Turso's stable VACUUM INTO. Lands in <base>/<label> where <base> is WEFT_BACKUP_DIR or <store_root>/backups and <label> is a traversal-guarded ?label= ([A-Za-z0-9][A-Za-z0-9._-]{0,99}: 1 to 100 letters, digits, ., _ or -, starting with a letter or digit and not ending with .) or, without one, a generated manual-<unix_millis>. The backup-<digits> form is reserved for the backup daemon (retention counts and prunes exactly those names), so a ?label= in that form is a 400, and no snapshot taken through this endpoint is ever pruned by retention. ?verify=source|snapshot picks the verification: source (the default) cross-checks each copy's user-table set + row counts against the live control plane — the strongest check, but it assumes a quiescent store; snapshot verifies each copy on its own terms (it reopens and fully scans every row of every table) without re-reading the source, which is what an online backup taken while ingest continues needs. An unknown token → 400. Returns per-DB {name, tables, rows, bytes} + total_rows/total_bytes + the verify mode that ran. The backup is built in a .partial-* directory beside its label and appears under the label, with a MANIFEST.json, only once complete and durable. A label outside that grammar → 400 (the grammar also keeps out .partial-*, .deleting-* and .restore-drill-*, the names of unfinished backups, prunes and drills); an existing target dir → 409 with "code": "already_exists"; no store → 503. Control plane only — the .weftseg measurement frames are not part of this backup (hard-constraint #3).
POST /api/v1/storage/restore/drill?label=<backup>Restore drill (Phase 7.4) — rehearse restoring a backup on a running server, so an operator can answer "is my backup actually restorable?" without risking anything. Restores the named backup into a throwaway directory beside the backups, verifies every restored database at its destination (it opens, and every row of every table reads), returns per-DB {name, tables, rows, bytes} + total_rows/total_bytes/restorable, then deletes the rehearsal copy (cleanup_error says why, if it could not; it is null otherwise). Non-destructive by construction — it cannot touch the live store, and restoring over a live control plane is deliberately not offered (the library primitive refuses to clobber; pointing WEFT_SEGMENT_STORE_ROOT at a restored copy is a deployment decision, not an HTTP call). Unknown backup → 404; a label outside the backup label grammar (so also traversal and the staging names .partial-*, .deleting-*, .restore-drill-*) → 400, while a daemon snapshot's backup-<digits> is accepted, since a drill only reads it; a rehearsal directory already taken by a drill started in the same millisecond → 409 with "code": "already_exists" (retry; nothing was restored or removed); a backup that will not restore → 500 carrying the failure, which is a failed drill rather than a server bug.
GET /api/v1/storage/{aspect}/stats · …/storage/statsMaterialized per-aspect and store-wide rollups, including the realized bytes/point (the north-star cost term), an unsorted_segments order-health count (segments that would force a linear scan on a point lookup), and an overlapping_segments count (time-overlapping segments — the cross-segment order-health signal, computed by an index scan). Rollup fields are served from the control plane without opening a segment.
\n

Metrics

\n

GET /metrics renders Prometheus text exposition (v0.0.4): shared\nweft_interpolate_* / weft_downsample_* counters across each endpoint family,\nweft_ingest_* counters (requests/errors/rows/segments) on the ingest paths,\nweft_reconcile_* counters (_passes_total / _segments_reconciled_total) over the\nout-of-order reconciliation passes (manual, store-wide, and the background daemon —\nthreshold-held calls excluded) plus _failed_passes_total, the per-aspect passes a\nstore-wide sweep (daemon tick or POST /api/v1/storage/reconcile) skipped after an\nerror (the daemon also logs one WARN line per failed aspect), weft_backup_*\ncounters (_snapshots_total / _bytes_written_total) over the online control-plane\nbackups, and latency histograms over the compute and seal paths.\nGET /debug/profile/current serves the same timing data as a live p50/p95/p99\nsnapshot.

\n

Backup & restore

\n

The control plane (the catalog, per-aspect schema, segment index and metadata\nrollup databases) has an online backup, two verification modes, an unattended\ndaemon, and a restore. Per the storage boundary this covers the control plane\nonly — the .weftseg measurement frames are not part of a snapshot.

\n
    \n
  • Online snapshot. POST /api/v1/storage/backup copies all four databases\nwith Turso's VACUUM INTO while the store stays fully usable, and verifies\nevery copy before returning (see the endpoint catalog above for the parameters).
    • \n
  • Two verifications, because "verified" means different things under load.\n?verify=source (default) cross-checks each copy against the live control plane\n— the strongest check, valid while the store is quiescent. ?verify=snapshot\nnever re-reads the source; it reopens each copy and fully scans every row of\nevery table, so it is correct while writers are committing. A snapshot-only\nbackup taken with six concurrent ingests in flight returns 200 with the\nconcurrent seals visible in the copy.
    • \n
  • A backup directory holds exactly its four databases and a MANIFEST.json.\nVerifying a snapshot reopens it, which leaves zero-byte -wal / -log journal\nsidecars beside the copy; these are swept after every verification, in both\nmodes. Only a sidecar that is exactly zero bytes is removed — a non-empty -wal\nholds un-checkpointed frames, which is data, and is never touched. Verification\nalso requires each database's own tables, so an empty file never passes.
    • \n
  • A backup appears only once it is complete and durable. It is built in a\n.partial-{label}-{nonce}/ directory; each database is verified and fsynced, the\nmanifest (format, creation time, each file's size, tables and rows) is written and\nfsynced, and the directory is fsynced, renamed to its label, and the parent\nfsynced. A backup that fails with an error before the rename removes its build\ndirectory again; a crash there leaves the .partial-* directory, which is never\ncounted or restored, and which the backup daemon's sweep removes (where the\ndaemon is not enabled, remove it by hand). Once renamed, only the parent's fsync\nis left: if that fails, the call reports an error (500 over HTTP) although the\ncomplete backup is already under its label, so a retry with the same label is\nrefused (409). The next backup's fsync of the same parent makes it durable.
    • \n
  • What a snapshot costs is dominated by the filesystem, not by the vacuum.\nBenchmarked in database/benches/backup_cost.rs:\non tmpfs a whole four-database backup-and-verify runs in 8.62 ms at one\naspect, 6.18 ms at 16 and 16.79 ms at 128 (snapshot verification;\nsource verification is within noise of those), so the vacuum's own work is\nmilliseconds and scales with control-plane rows rather than being a fixed\nper-database floor. On a real disk it is orders of magnitude slower, because\nTurso's VACUUM INTO fsyncs the destination and runs a TRUNCATE checkpoint\nbefore returning. On this project's btrfs volume (load average ~12) the\nbackground daemon's ticks landed 3.0–4.5 s apart at a 2 s configured interval.\nA tick fires late only when the one before it outlasts the interval (tokio's\ndefault Burst catch-up), so each snapshot and its prune took that long, and\nthey ran back to back. Set WEFT_BACKUP_INTERVAL_SECS from what your storage\ncan sustain, not from how much data you have; point WEFT_BENCH_BACKUP_DIR\nat your own volume to measure it.
    • \n
  • Unattended backups with retention. Set WEFT_BACKUP_INTERVAL_SECS to run a\nbackground daemon that snapshots into a fresh backup-<unix_millis> directory\neach tick (verifying snapshot-only, since it backs up a live store), and\nWEFT_BACKUP_KEEP to retain only the newest N. Retention is deliberately narrow:\nit only ever removes daemon-generated backup-<digits> directories, a form\nthe API refuses to create (a directory you rename into it by hand is treated as\ngenerated and can be pruned, as it always could), so a snapshot you took\nthrough the API (?label=nightly, or an unlabelled manual-<unix_millis>) is\nnever a prune candidate and never counts toward N, and\nit runs only after a successful snapshot, so a run of failures cannot prune\nyour last good backup away. It counts only complete backups (a manifest, or all\nfour databases from before manifests), and renames a pruned backup to\n.deleting-* before removing its files, so an interrupted prune never leaves a\nhalf-removed backup. A backup-<digits> directory stamped more than 24 h\npast the clock, or past the directory's own modification time, did not come from\nthe daemon: retention neither counts nor removes it, and logs a warning each time\nit sees it. The modification-time check keeps such a directory ignored after the\nclock catches up with its stamp, as long as the directory is left unmodified and the\nfilesystem reports modification times; remove it by hand when the warning appears.\nAt start and on every tick the daemon removes .partial-*, .deleting-* and\n.restore-drill-* entries untouched for an hour. Both paths record\nweft_backup_* metrics.
    • \n
  • Restore, with a drill. weftdb::restore_control_plane(backup_dir, root)\nputs a snapshot back into a store root and verifies each restored file at its\ndestination — what matters is that the file the store will open actually\nreads. It refuses an incomplete backup directory (or one whose files no longer\nmatch its manifest, or one with a staging name) and refuses to overwrite an\nexisting control plane, pre-flighting both across all four files before writing\nanything, so a rejected restore leaves nothing behind. Each file is copied to\n<name>.tmp, fsynced and verified; only once all four verify are they renamed\ninto place, one after another, and the root fsynced. A failure before the\nrenames leaves only .tmp files, which a retry replaces; one part-way through\nthem leaves some databases in place, which a retry refuses to overwrite, so clear\nthe root first. Restoring beside the\noriginal segments/ frames reconstitutes a working store: the round-trip test\nreopens the restored root and reads its measurements back.
    • \n
  • Drills on a live server. POST /api/v1/storage/restore/drill?label= rehearses\na restore of the named backup into a throwaway directory, verifies it, reports\nrestorable, and cleans up — non-destructive by construction, so the question\n"is my backup actually restorable?" can be asked of production. Restoring over\na live control plane is not offered: that is a deployment decision (point\nWEFT_SEGMENT_STORE_ROOT at a restored copy and restart), not an HTTP call.
    • \n
\n
\n

Configuration

\n

WeftDB is configured primarily through environment variables:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
VariableUsed byPurposeDefault
WEFT_DATA_DIRweftdb, weft-tuiRoot directory for database files and the TUI log (weft-tui.log).portable per-user default (see below)
TEST_DATA_DIRweftdbHighest-priority override for the database root (used by the test suite).unset
WEFT_EXTRACTED_BATCH_RETENTION_SECSweftdbHow long (seconds) a legacy aspect remembers the batches pattern extraction consumed, so the incremental build does not queue them again. Each extraction deletes older records, keeping about one row per resolution step of this period. Set it above the longest interval between pipeline runs of an aspect plus the longest ingest call; a non-positive or unparsable value keeps the default.172800 (48 hours)
WEFT_SERVER_ADDRweft-serverHTTP bind address.127.0.0.1:8080
WEFT_ALLOW_ANY_HOSTweft-serverTruthy (1/true/yes/on) → turn off the loopback request guard. On a loopback bind the server otherwise answers only requests whose Host is localhost, 127.0.0.0/8 or [::1] (421 otherwise) and refuses state-changing requests from non-loopback web origins (403). Set it behind a local reverse proxy that forwards a different Host, or one that forwards the non-loopback Origin of a browser UI it fronts. A non-loopback bind is never guarded.unset (guarded on a loopback bind)
WEFT_GPU_CALIBRATEweft-serverStartup calibration. By default, once the listener is bound, the server runs splimes::calibrate() once in the background on a blocking thread: it starts the GPU if there is one, times the single-thread, rayon and GPU backends on grids up to 16 Mi points (several seconds, a few hundred MB), and sets where Backend::Auto switches between them. Requests are served from the start and interpolate on the CPU with splimes' default thresholds until it finishes. A CPU/software adapter (llvmpipe, lavapipe, WARP) is not calibrated, with a log line saying why; force calibrates it anyway. 0 (or false/no/off) skips calibration. It logs the adapter (or why there is none) and the thresholds, and never fails startup; skipped or failed, interpolation stays on the CPU with splimes' defaults.unset (calibrate, except a software adapter)
WEFT_MAX_INTERPOLATE_POINTSweft-serverThe most output points one /api/v1/interpolate* request may produce; a larger grid is a 400 naming its size and the limit, refused before anything is allocated. A positive integer, read once at startup; anything else (including 0) stops the server from starting with a message naming the variable.10000000
WEFT_SEGMENT_STORE_ROOTweft-serverRoot of the Storage v2 segment store; enables the /storage endpoints.unset (storage endpoints answer 503)
WEFT_ON_AMBIGUOUS_COMMITweft-serverWhat the server does once its segment store is write-poisoned (after a COMMIT that may or may not have committed): poison keeps serving, refusing every write until the server restarts while reads keep working and GET /ready reports poisoned; exit logs and exits with status 70, for a supervisor that restarts the server. The restart's open settles the transaction either way. Read once, at startup; the weftdb library never reads it and never exits the process.unset (poison)
WEFT_RECONCILE_INTERVAL_SECSweft-serverBackground reconcile daemon sweep interval in seconds; 0/unset disables it.unset (disabled)
WEFT_RECONCILE_THRESHOLDweft-serverunsorted_segments backlog an aspect must reach before the daemon reconciles it.1
WEFT_RECONCILE_HOT_COLDweft-serverTruthy → the daemon reconciles cold segments each tick and defers the hot tail until the threshold.unset (all-or-nothing)
WEFT_RECONCILE_OVERLAPSweft-serverTruthy → each daemon tick also merges cross-segment time-overlap groups.unset (disabled)
WEFT_RECONCILE_SPLIT_MIN_BYTESweft-serverSplit-not-rewrite floor (bytes) for the daemon's overlap merge; a dominant cold prefix clearing it is split off rather than rewritten.unset (default 50 MiB floor → full-rewrite)
WEFT_RECONCILE_MAX_SPLITSweft-serverSegment-count cap; each tick also squashes every aspect over it into one segment (bounds split-path fragmentation).unset (no squash)
WEFT_COMPACT_TARGET_ROWSweft-serverTarget segment size in rows for the daemon's size-aware compaction pass; each tick coalesces every aspect's segments toward ~this many rows per segment (leaving already-large segments alone), holding fragmentation near the read-optimal size instead of folding to one (WEFT_RECONCILE_MAX_SPLITS). Motivated by the downsample_range knee (one giant segment reads slower than several mid-sized ones). Needs the daemon interval set.unset (no compaction)
WEFT_BACKUP_DIRweft-serverDirectory each control-plane backup is written under — shared by the manual POST …/storage/backup endpoint and the background backup daemon, so hand-taken and automatic snapshots live together.<store_root>/backups
WEFT_BACKUP_INTERVAL_SECSweft-serverBackground control-plane backup daemon interval in seconds; 0/unset disables it. Each tick writes a verified snapshot into a fresh backup-<unix_millis> directory, using the concurrent-write-safe snapshot verification (it backs up a live store). Needs a segment store configured.unset (disabled)
WEFT_BACKUP_KEEPweft-serverHow many daemon-generated snapshots to retain; after each successful backup the oldest backup-<digits> directories beyond this many are removed. Only daemon-generated names are ever counted or pruned — a snapshot taken through POST …/storage/backup (a ?label= one, or an unlabelled manual-<unix_millis> one) is never a candidate — and a generated-looking directory stamped more than 24 h past the clock or past its own modification time is ignored (not counted, not removed, logged as a warning; remove it by hand).unset (retain everything)
BTC_TEST_MAX_ROWS / BTC_TEST_FULLweftdb (tests)Row cap for the bounded, hermetic default run of test_create_btc_1min_database (into a temp data dir), or BTC_TEST_FULL=1 for the historical whole-corpus load into the shared data dir.5000 / unset
WEFT_SEGMENT_CHECKPOINT_STRIDEweft-serverRows between entries of the sealed timestamp checkpoint index — trades a little size for much faster point lookups on sorted, irregular columns (~3.45× single-block; see the checkpointed-frames feature above). Applies only where it pays: sorted + irregular + at least WEFT_SEGMENT_CHECKPOINT_MIN_ROWS rows. ~1024 is the sweet spot (stride barely moves speed but does move size).unset (no index; frames byte-for-byte as before)
WEFT_SEGMENT_CHECKPOINT_MIN_ROWSweft-serverRow floor below which a segment is never checkpointed (a small column decodes trivially, so an index would be pure cost).8192
WEFT_SEGMENT_CHECKPOINT_MAX_CODEC_OVERHEADweft-serverCeiling on the timestamp-codec override a checkpointed seal will accept (blocked / best bytes; 1.0 = only when free). A checkpointed frame must use the range-decodable per-block codec, which is ~free where that codec already wins but ~3.5× on a Gorilla-shaped and ~14× on an RLE-shaped column — this refuses those seals rather than silently bloating them.1.25
WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEADweft-serverCeiling on the size overhead the bit-sliced (transposed) value codec may pay against the size-selected codec (transposed / best bytes; 1.0 = only when free, and a value below 1.0 is meaningful — the bit-sliced layout can be a strict size win, since it pays one width header per 1024-value tile where the blocked codec pays one per 64 values). Stores the value column bit-plane-major. Measured byte-neutral end to end on a 1M-row column and no faster to read than the linear codec (see Where it doesn't pay off), so it is off by default. Needs a build with the bitsliced-codec feature; any other build ignores the variable and logs a warning.unset (no transposed codec; frames byte-for-byte as before)
WEFT_SEGMENT_PARTIAL_BASEweft-serverResolution token (seconds/minutes/hours/…) at which each sealed segment materializes a partial-reduction .weftpart sidecar — a stored mergeable partial of the bounded reductions. A stored-range downsample of those reductions then merges the sidecars instead of decoding the value column (measured 3.2×), re-keyed to any coarser nesting resolution. A sealed segment is immutable, so the sidecar never goes stale; a rewrite (reconcile/split/squash) regenerates it.unset (no sidecar; downsample_range decodes as before)
WEFT_SEGMENT_PARTIAL_MIN_ROWSweft-serverRow floor below which a segment gets no partial sidecar (a tiny segment's partial saves too little decode to be worth the extra file).4096
WEFT_SEGMENT_PARTIAL_TIERSweft-serverComma-separated fine→coarse rollup resolutions (e.g. hours,days) materialized beside the WEFT_SEGMENT_PARTIAL_BASE partial, each re-keyed from the tier below (up to four kept; a finer/non-nesting entry is skipped). A coarse stored-range downsample then folds the coarsest matching tier instead of re-keying the whole fine base (measured ~1.16× on a DAY query over a MINUTES base). No effect unless WEFT_SEGMENT_PARTIAL_BASE is set.unset (base-only sidecar)
RUST_LOGalltracing filter. On weft-server it drives a per-request root span (request{method,path,request_id}, echoed as x-request-id) that every per-stage span nests under, each with busy/idle timing: the compute paths (interpolate.parse/compute/serialize under interpolate.engine, downsample.parse/reduce); the storage read paths (storage.{range,value_range,point}.read + .serialize, with a format field over JSON/CSV/Arrow/Parquet); the ingest paths (storage.ingest.parse/normalize/seal for ILP/CSV/JSON, and storage.ingest.parquet for the Parquet decode+seal); the background reconcile daemon (reconcile.tick{kind,…,aspects,segments}); and the control-plane writes on the seal path (control_plane.index.insert{aspect,id,rows_changed}, control_plane.index.delete{…}, control_plane.metadata.put{aspect,rows_changed}), whose rows_changed is libSQL's own affected-row count (Statement::n_change()) rather than an inference — so a trace shows whether a catalog write actually changed anything.weft_tui=debug,database=debug,info
OTEL_EXPORTER_OTLP_ENDPOINTweft-serverWhen set (e.g. http://localhost:4317), export tracing spans to an OpenTelemetry collector over OTLP/gRPC in addition to the RUST_LOG fmt output. Unset → no exporter, no network dependency; a misconfigured/absent collector never blocks startup. scripts/verify-otlp.sh verifies delivery end-to-end against a local Jaeger container (starting one if needed).unset (export disabled)
SKIP_SLOW_TESTStestsSet to 1 to skip long-running tests.unset
\n

The WEFT_SEGMENT_* variables configure the store weft-server opens; the server maps\nthem to weftdb::SegmentStoreOptions. A program embedding weftdb gets them only by\nasking: SegmentStore::open uses SegmentStoreOptions::default() (every one of them\nunset), and SegmentStore::open_with_options(root, SegmentStoreOptions::from_env())\nreads them as the server does.

\n

The database root directory is resolved in this order:

\n
    \n
  1. TEST_DATA_DIR, if set (explicit override for the test suite);
    2. \n
  2. otherwise WEFT_DATA_DIR, if set;
    3. \n
  3. otherwise a portable, per-user default — ~/.weftdb/data (falling back to the\nplatform data directory + weftdb, e.g. %APPDATA%\\weftdb on Windows, if the home\ndirectory cannot be determined).
    4. \n
\n

No paths are hard-coded: data lands in a writable, machine-independent location out\nof the box, and setting WEFT_DATA_DIR relocates both the databases and the TUI log\ntogether. The resolution functions are exported as weftdb::data_dir() (resolved)\nand weftdb::default_data_dir() (the raw default).

\n

The splimes crate also exposes build features:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
FeatureEffect
gpu (default)The wgpu backend. Without it, interpolation always runs on the CPU.
serde (default)Serialize/Deserialize for Point, PointKind, Resolution and Spline.
tokioInterpolator::run_async, which runs an interpolation on tokio's blocking pool. WeftDB enables it: every async caller (the weftdb read and compression paths, weft-server, the bench adapter) interpolates off the async workers.
\n

The GPU is started at runtime, not at build time: weft-server calibrates it in the\nbackground at startup (WEFT_GPU_CALIBRATE, above), and an embedding program calls\nsplimes::calibrate() or splimes::prewarm_gpu() itself.

\n

WeftDB's own build features are all off by default, sit outside the 1.0 semver\npromise, and are pending patent review. A default build reads every segment written\nunder the default configuration (WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEAD unset). A segment\nwritten with that variable set, as builds from before v0.1.0 did whenever it was set, needs\nbitsliced-codec. The release binaries are built without either feature, except that\nweft-bench enables experimental-codecs.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
FeatureCrate(s)Effect
bitsliced-codecweft-physical-type, forwarded by weftdb and weft-serverCompiles in the opt-in bit-sliced value codec, so WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEAD takes effect. Without it, a segment written with the codec fails to read with an error naming the feature.
experimental-codecsweft-physical-type (enabled by weft-bench)Advisory codecs never written to disk: Gorilla-XOR, Chimp, Chimp128 and Elf for f64, and the Sprintz FIRE timestamp forecaster. Benchmark what-ifs only.
\n
\n

Rust library

\n

WeftDB can be used directly as a set of libraries. They are not on crates.io yet\n(v0.1.0 ships binaries only), so depend on them from this repository, pinned to a\nrelease tag. splimes, the interpolation engine, is on crates.io:

\n
[dependencies]\n# the database: capture, query, interpolate\nweftdb = { git = \"https://github.com/basic-automation/weftdb\", tag = \"v0.1.0\" }\n# the analytics pipeline\nweft-orchestration = { git = \"https://github.com/basic-automation/weftdb\", tag = \"v0.1.0\" }\n# the interpolation engine on its own\nsplimes = \"1\"
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
CrateDepend on it when you want
weftdbThe database — capture measurements, query and interpolate stored series.
splimesSpline interpolation over irregular series, standalone. No database.
weft-orchestrationThe batching → patterns → events → correlation → signals pipeline.
weft-physical-typeDeclared numeric encodings and the .weftseg columnar format.
weft-reduceDownsampling reductions over an epoch-aligned bucket grid.
weft-line-protocolTo parse InfluxDB Line Protocol without an InfluxDB client.
weft-arrowSegments as Apache Arrow RecordBatch, Arrow IPC or Parquet.
weft-arrow-storeA stored range read straight into an Arrow RecordBatch.
\n

weft-server, weft-tui and weft-bench are binaries rather than libraries, so they\nare not published to crates.io — take them from the archives on a\nGitHub release or build them\nfrom this repository.

\n

The examples below are illustrative — run cargo doc --open for the authoritative,\nversion-matched API.

\n

Capture Measurements

\n
use weftdb::{Database, DatasetId, InputMeasurement, Resolution};\nuse bigdecimal::BigDecimal;\nuse std::str::FromStr;\nuse chrono::{TimeZone, Utc};\n\n#[tokio::main]\nasync fn main() -> anyhow::Result<()> {\n    let db = Database::new(\"my_sensors\").await?;\n    let sensor = db.observe_subject(\"temperature_sensor_001\").await?;\n    let temp = db.track_aspect(&sensor.id(), \"ambient_temp\", &Resolution::Seconds, None).await?;\n\n    let measurements = vec![\n        InputMeasurement::new(Utc.with_ymd_and_hms(2024, 1, 1, 12, 0, 0).unwrap(), BigDecimal::from_str(\"22.5\")?),\n        InputMeasurement::new(Utc.with_ymd_and_hms(2024, 1, 1, 12, 1, 0).unwrap(), BigDecimal::from_str(\"22.8\")?),\n    ];\n    db.batch_capture_measurements(temp.id(), DatasetId::new(), measurements).await?;\n    Ok(())\n}

Query & Interpolate

\n
use weftdb::{Database, Resolution, Spline};\nuse chrono::{TimeZone, Utc};\nuse futures::StreamExt;\n\nlet db = Database::existing(\"my_sensors\").await?;\n\n// Interpolated point at a specific time:\nlet point = db.analyze_point(&aspect_id, query_time, &Resolution::Seconds, &Spline::Linear).await?;\n\n// Streamed, interpolated range:\nlet mut stream = db.analyze_range(&aspect_id, start, end, Resolution::Seconds, Spline::Linear).await?;\nwhile let Some(result) = stream.next().await {\n    let p = result?;\n    println!(\"{}: {}\", p.timestamp, p.value);\n}

Run an Analytics Pipeline

\n
use weft_orchestration::Pipeline;\nuse weftdb::Database;\nuse splimes::Spline;\n\nlet database = Database::existing(\"PlantTelemetry\").await?;\n// ... resolve `aspect_id` for the \"outlet_pressure\" aspect of pump-station-3 ...\n\nlet mut pipeline = Pipeline::builder(database.clone(), aspect_id)\n    .spline_method(Spline::Linear)\n    .batch_size(24)                              // 24-hour batches\n    .add_dictionary(\"daily\", \"Daily patterns\", constraints)\n    .with_monthly_increase_detector(0.05)        // 5% monthly increase\n    .with_peak_detector(\"Pressure Peaks\")\n    .build()\n    .await?;\n\npipeline.run().await?;                            // full pipeline (incremental)\nprintln!(\"Signals generated: {}\", pipeline.signals().len());

For finer control, the individual steps — prepare_data, extract_patterns,\ndetect_events, correlate_events, generate_signals — can be called directly.

\n
\n

Terminal UI

\n

weft-tui is the interactive way to explore WeftDB (the HTTP server is the primary\ncommercial surface; the TUI serves demos and debugging). Launch it with\ncargo run --release -p weft-tui. It is a stateful, keyboard-driven application that\nwalks you through:

\n
    \n
  • Database / Subject / Aspect creation — with name validation and resolution\nselection (+/- to step through resolutions from nanoseconds to years).
    • \n
  • CSV import — point it at a CSV file and watch live loading progress\n(rows loaded / scanned).
    • \n
  • Browsing — select an existing database, subject, and aspect.
    • \n
  • Plotting — visualize an aspect's series directly in the terminal.
    • \n
  • Compression — choose a mode (time-based, size-based, or combined), configure\ntiers/targets/aggressiveness in a form, and monitor a live compression dashboard\n(phase, tier progress, aggressiveness, compression ratio) plus last-run stats.
    • \n
\n

Logs are written to weft-tui.log (see WEFT_DATA_DIR), and the last 50 log lines\nare also shown in-app.

\n
\n

How it works

\n

The parts below are reference material — you do not need them to use WeftDB.

\n

Architecture

\n
┌──────────────────────────────┐  ┌──────────────────────────────────────┐\n│           weft-tui            │  │              weft-server               │\n│  Ratatui terminal UI (app)   │  │  axum HTTP API: interpolate ·         │\n│  create DB · import · plot   │  │  downsample · storage ingest/query ·  │\n│  · compress                  │  │  catalog · /metrics · /debug/profile  │\n└──────────────┬───────────────┘  └──────┬───────────────────┬───────────┘\n               │ uses                     │ uses              │ columnar I/O\n               ▼                          ▼                   ▼\n┌──────────────────────────────┐   ┌────────────────────────────────────┐\n│    weft-orchestration    │   │   weft-arrow / weft-arrow-store      │\n│  batch → patterns → events   │   │   Arrow IPC + Parquet interchange  │\n│  → correlations → signals    │   └──────────────────┬─────────────────┘\n└───────────────┬──────────────┘                      │\n                ▼                                     ▼\n┌─────────────────────────────────────────────────────────────────────┐\n│                              database                                │\n│   Turso/libSQL control plane (catalog · metadata · segment index)   │\n│   + typed columnar `.weftseg` segment store (measurement hot path)   │\n│   Subject/Aspect/Measurement · Patterns · Events · Compression      │\n└───────────────┬─────────────────────────────────────┬───────────────┘\n                │ physical types + segments           │ numerics\n                ▼                                     ▼\n┌──────────────────────────────┐   ┌──────────────────────────────────┐\n│      weft-physical-type       │   │             splimes              │\n│  encodings · schemas ·       │   │   Spline interpolation engine    │\n│  timestamp codecs · .weftseg  │   │  CPU · Rayon · GPU (calibrated)  │\n└──────────────────────────────┘   └──────────────────────────────────┘

weft-bench drives the whole stack from outside through the same public APIs a\ncustomer would use; weft-line-protocol is the shared ILP dialect between the\nharness and the server. Dependencies flow downward; the arrow-* tree lives only\nin the weft-arrow* leaf crates.

\n
\n

Data Model

\n

WeftDB organizes data hierarchically. Each level maps to a directory or database file\non disk, so an aspect's storage is fully self-contained.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ConceptDescriptionExample
DatabaseTop-level container for all data.Crypto, my_sensors
SubjectA logical entity being observed.pump-station-3, temperature_sensor_001
AspectA specific measurement type of a subject, with its own resolution, declared physical encoding, and (optional) compression config.open, ambient_temp
MeasurementA timestamped, arbitrary-precision value.(2024-01-01T12:00Z, 22.5)
\n
On-Disk Layout
\n
{data_dir}/\n└── {database_name}/\n    ├── metadata.db                  # database metadata\n    └── {subject_name}/\n        └── {aspect_name}/\n            ├── measurements.db        # raw measurements\n            ├── unprocessed_batches.db # batching queue\n            ├── processed_batches.db\n            ├── patterns.db            # extracted patterns\n            ├── events.db              # detected events\n            ├── correlations.db        # pattern↔event links\n            ├── pipeline.db            # persisted pipeline config & state\n            └── dictionaries/\n                └── {dictionary_name}.db

The Storage v2 segment store (see\nPhysical Types & Columnar Storage) adds, per\nstore root: catalog.db (database/subject/aspect hierarchy + declared schemas),\nper-aspect metadata.db rollups, segment_index.db (the prune-before-open\nsegment index), and segments/*.weftseg (the typed columnar measurement files).

\n

Higher-level analytics types build on this foundation:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
TypePurpose
BatchA group of measurements processed together.
PatternAn extracted recurring shape, with the occurrences where it appears.
DictionaryA collection of patterns sharing similarity constraints.
EventA detected occurrence with one or more manifestations (time spans).
CorrelationA link between a pattern and an event.
SignalA probability-weighted prediction derived from a correlation.
\n
\n

Interpolation Engine

\n

splimes is the numerical heart of WeftDB. Given a set of Points and a target time\nrange + resolution, it produces an interpolated series using one of four spline\nmethods, choosing the execution backend automatically, and labels every output point\nRaw, Interpolated or Extrapolated (PointKind). It interpolates and\nextrapolates — single-instant lookups and full ranges — on the CPU (one thread or\nrayon's pool) and the GPU. WeftDB depends on splimes 1.0; its own\nmigration guide\nlists what changed from 0.1.

\n
Spline Methods
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
MethodMin. pointsNotes
Linear2Straight-line interpolation.
Quadratic3Second-degree.
Cubic4Smooth third-degree splines.
Polynomial(degree, bounds_factor)degree + 1Degree 1–8, with optional bounds damping of the extrapolation.
\n

If a method needs more distinct points than are available, splimes steps down\nautomatically (Cubic → Quadratic → Linear; Polynomial(d) → Polynomial(n − 1)),\nnever upgrading beyond what you requested, and reports the method it used.

\n
Backend Selection
\n

Every call is synchronous; Backend::Auto (the default) picks the backend by output\ngrid size, using AutoThresholds:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Grid sizeBackend
below parallel_min_points (default 64 Ki)CPU, the calling thread
from parallel_min_pointsParallel, rayon's pool
from gpu_min_points (default never), once the GPU has been startedGPU (wgpu compute shaders), falling back to rayon if it fails
\n

Auto never starts the GPU itself. splimes::calibrate() starts it, times every\nbackend on this machine and sets the thresholds to the measured crossovers;\nweft-server runs it once in the background at startup, skipping a CPU/software\nadapter (WEFT_GPU_CALIBRATE=0 skips it, force calibrates any adapter). Otherwise:

\n
    \n
  • splimes::prewarm_gpu() starts the GPU and returns its GpuInfo (adapter, API,\ndevice type, driver, f64 support); then set thresholds with\nsplimes::set_auto_thresholds.
    • \n
  • splimes::configure_gpu(GpuConfig) — before the GPU's first use — sizes its\nbuffer pool and per-dispatch chunk (max_pool_bytes, chunk_points, low_power;\npresets GpuConfig::minimal(), low_memory(), DEFAULT, high_performance()).\nCalling it after the GPU has started is an Error::GpuAlreadyConfigured, never a\nsilent no-op; splimes::gpu_config() reports what is in force.
    • \n
  • splimes::gpu_pool_stats() returns the buffer pool's counters once the GPU is up.
    • \n
  • The GPU computes in f64 where the adapter supports it; f32 must be asked for\n(Interpolator::gpu_precision(Precision::F32)).
    • \n
\n
use splimes::{Interpolator, Resolution, Spline};\n\n// Backend::Auto picks the backend by grid size; from async code, run it off the\n// executor (`run_async` needs the `tokio` feature):\nlet series = Interpolator::new(Spline::Cubic, Resolution::Seconds).run_async(points, start, end).await?;\nfor (timestamp, value, kind) in series.iter() {\n    // kind: PointKind::Raw | Interpolated | Extrapolated\n}
\n

Physical Types & Columnar Storage

\n

WeftDB's storage hot path is its own typed columnar format; Turso/libSQL serves as\nthe control plane (catalog, metadata, segment index, pipeline state) and never\nholds bulk measurements. BigDecimal remains the logical/API type everywhere.

\n
Physical type system (weft-physical-type)
\n
    \n
  • Six numeric encodings — F64, F32, ScaledI64, ScaledI128,\nDecimal128, BigDecimalText — each declaring storage width, hot-path/GPU\neligibility, and conversion behavior. Every encode reports an explicit\nExactness (exact / lossy-with-measured-residual); to_logical is always\navailable.
    • \n
  • No silent downcast, enforced — AspectSchema declares, per aspect, the\nphysical encoding, a permitted per-value error bound, and the timestamp unit.\nSealing a segment under a declared schema errors (SealError::Encode /\nSealError::ToleranceExceeded) rather than losing precision beyond the\ndeclared bound. Over HTTP this surfaces as a 400 — never a downcast.
    • \n
  • Advisory encoding selection — recommend_encoding picks the narrowest safe\nencoding within a tolerance, and columnar estimators report expected\nbytes/point (the north-star cost term) before anything is written.
    • \n
  • Timestamp codecs — integer-epoch timestamps (TimeUnit =\nseconds/millis/micros/nanos) with lossless delta and delta-of-delta\ntransforms and five interchangeable second-difference codecs — per-value\nzig-zag + LEB128 varint, RLE, fixed-width bit-packing, a\nGorilla-style variable-length codec, and per-block adaptive (dynamic)\nbit-packing — chosen per column by a smallest-wins selector. Each wins a different\nregime: bit-packing on regular/small-jitter series, RLE on long constant runs, Gorilla\non scattered single jitter (isolated moderate spikes among regular intervals, where\nRLE cannot form runs and bit-packing must widen every value), and per-block adaptive\nbit-packing on a mixed-magnitude stream (a contiguous wide region among narrow runs,\nwhere a single global width overpays — 184 B vs 384–961 B for the other codecs on a\n256-value mixed corpus). A regular 1000-point column packs to ~12 bytes total.
    • \n
  • Value-column codecs — a value column is stored under its schema-declared physical\nencoding, and a ScaledI64 column additionally chooses, via a self-describing\nselector byte, between a per-value zig-zag varint, fixed-width bit-packing of its\nmantissas, per-block adaptive (blocked) bit-packing, and per-block\nFrame-of-Reference (FOR) packing — bit-packing a regular scaled series (e.g. a\n2-decimal price ramp) to 37% below the varint, the blocked codec confining a wide\nburst to the blocks it spans on a mixed-magnitude column, and FOR (subtract each\nblock's minimum, pack the unsigned residual to the block's range — the standard\nFastLanes/ALP move) collapsing values clustered at a high base (a sensor reading\nnear a fixed offset: mantissas near 1e9 store 88% below the bit-packed\nalternative). Each codec is chosen only when strictly smallest, so a regular column is\nbyte-for-byte unchanged; the realized figure is reported as\nStorageEstimate.realized_value_bytes / value_codec. The timestamp column keeps an\nadvisory FOR estimate (for_estimated_bytes) — second differences are near-zero, so\nFOR rarely wins there. Every linear bit-packed codec decodes with a word-wise field read\n(one unaligned load and a shift/mask per value), so 1 Mi mantissas unpack in 1.09 ms\n(fixed width) / 1.83 ms (per-block)\n(weft-physical-type/benches/bitunpack.rs), and\n1 Mi real BTC closes stored under FOR at scale 8 decode to floats in 1.66 ms\n(weft-physical-type/benches/alp_vs_f64_codecs.rs).
    • \n
  • Two-level delta cascade (opt-in) — a trending ScaledI64 column (a counter or\nmonotone sensor whose magnitude every single-level codec pays for) is additionally\ncompressible by a cascade: delta-transform the mantissas, then pack the differences with\nthe smallest inner packer (varint / bit-pack / blocked / FOR / RLE). It is realized on disk\nas the VAL_CODEC_DELTA_CASCADE codec-chain descriptor (write_value_column_cascading,\nread back by the ordinary reader) and surfaced advisory-first in the bench\n(StorageEstimate.advisory_delta_cascade_value_bytes, schema v15). It is opt-in — the\ncascade beats even FOR broadly, so folding it into the default selector is a headline change\nheld for owner sign-off; the default codec choice is unchanged.
    • \n
  • Bit-sliced (transposed) value codec (opt-in, bitsliced-codec build feature) —\nVAL_CODEC_TRANSPOSED stores a\nScaledI64 column's mantissas bit-plane-major in 1024-value tiles, so the decoder rebuilds\neight lanes per plane byte with branch-free table spreads and a tile's width header drops the\nempty high bit-planes of a small-magnitude column. The decode kernel is checked against an\nexternal yardstick, the published fastlanes crate, at equal bit width\n(weft-physical-type/benches/fastlanes_yardstick.rs,\n1 Mi values): WeftDB's decoder takes 0.78 / 1.50 / 2.32 ms at widths 3 / 10 / 20 against\nfastlanes' 0.33 ms — still 2.4–7× behind, and that gap is an open roadmap item, not a\ndesign choice. It carries its own size function (transposed_value_bytes) and selector\nentry (best_value_codec_transposed(max_overhead)) rather than reusing the blocked figure, is\nrandom-access capable (so it does not regress the streaming point read), and is requested through\nFrameOptions / Segment::write_to_with or, from a deployment, WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEAD.\nHonest result: no read-path win over the linear codec. Measured on a 1M-row zero-straddling\ncolumn (weft-physical-type/benches/transposed_read.rs,\n2026-10-08, transposed vs linear): frame bytes +0.08% (1,251,057 vs 1,250,076), full decode\n7.81 ms vs 6.87 ms; with the slicing-by-16 frame CRC, windowed 1000-row range 638 µs vs\n632 µs and single point read 820 µs vs 805 µs. So the layout is byte-neutral and no faster\n(slightly slower) on reads; it stays opt-in and the\ndefault codec choice is unchanged.\nNote the codec is chosen only when a strict size win or within the caller's overhead ceiling —\nthe ceiling may legitimately be set below 1.0, because per-tile widths with one header per 1024\nvalues can beat both a global width and the blocked codec's one-header-per-64. It is a\nbit-sliced layout, not the FastLanes layout (which keeps each value's bits together), and it\nis compiled only with the bitsliced-codec feature, pending patent review; without it the\nwriter never selects it and reading a segment that uses it fails with an error naming the\nfeature.
    • \n
  • Block-level random access — the fixed-layout value codecs support decoding a single value\n(or a sub-range) without materializing the whole column: weftseg::read_value_at(bytes, i)\nreads only the block covering row i (skipping earlier blocks by their headers) for the\nblocked/FOR codecs, reads bit i * width directly for the fixed-width bit-pack codec, and decodes\nonly the covering tile for the bit-sliced codec — the point-lookup / late-materialization lever.\nweftseg::read_value_range(bytes, start, len) is its windowed sibling: every fixed-layout codec\nlocates a value by walking its block/tile headers from the start of the stream, so resolving a\nwindow one value at a time re-walks that chain per row. The range read does the decode once,\nwhich is what keeps a windowed read on the 1024-value-tile bit-sliced layout from costing a whole tile\ndecode per row. weftseg::read_segment_point(bytes, t)\nwires this up to the framed single-block segment — it skips the value block by its framing,\ndecodes only the timestamps to find the row, and unpacks the one covering value block — so a\npoint lookup never materializes the value column on a per-block codec (equal to a full\ndecode + value_at for every frame; the non-block codecs fall back to that). read_paged_segment_point\ndoes the same for a paged frame, first pruning pages on their indexed min/max timestamp so only\nthe surviving page is touched. A regular (constant-stride) timestamp column is resolved in\nclosed form — ts[i] = first + i*step, so the row for an instant is O(1) with no timestamp\nmaterialization at all. Measured ~46× faster point lookup on a 100k-row single-block FOR\nsegment (56 µs vs 2.60 ms) and ~8.5× faster on the paged frame (33 µs vs 279 µs), with the\nclosed-form timestamp path a further ~3.4× over an irregular column (56 µs vs 188 µs), identical\nbytes on disk (weft-physical-type/benches/pointread.rs).
    • \n
  • Checkpointed frames — sublinear point lookup on an irregular sorted column (opt-in;\nwrite_segment_checkpointed / write_paged_segment_checkpointed). The closed form above only\nserves a regular column; an irregular one has no closed form, so a probe had to decode the whole\ndelta-of-delta timestamp stream. A checkpointed frame persists a sparse (row, timestamp, delta)\nindex every stride rows next to a range-decodable per-block dod stream, so a probe binary-searches\nthe index and decodes only the blocks it walks — the timestamp column is never materialized.\nMeasured ~3.45× faster point reads on a 100k-row sorted-irregular frame (352.94 ms → 102.42 ms\nfor 64 probes) for +0.41% frame bytes at stride 1024; a paged frame gains only ~1.14×\n(68.70 → 60.41 ms), because page pruning already bounds its decode to rows_per_page\n(weft-physical-type/benches/dodsearch.rs). The codec tag\nis additive — every previously written frame still reads, with no format-version bump — and a\ncheckpointed frame is byte-for-byte equivalent in behaviour to a plain one on every read.\nOpt in by setting WEFT_SEGMENT_CHECKPOINT_STRIDE (see the configuration table). The seal path\nthen writes the index only where it genuinely pays, on three counts: the column is sorted and\nirregular (a regular one already resolves O(1) closed-form, an out-of-order one cannot be\nbinary-searched), it has at least WEFT_SEGMENT_CHECKPOINT_MIN_ROWS rows, and the codec override\nis affordable — a checkpointed frame must use the range-decodable per-block codec, which costs\n~3.5× on a Gorilla-shaped column and ~14× on an RLE-shaped one, so those are refused rather than\nbloated (WEFT_SEGMENT_CHECKPOINT_MAX_CODEC_OVERHEAD). It is off by default, so an unconfigured\nstore's bytes/point are unchanged; making it the default is an open roadmap decision.
    • \n
  • Realized headline bytes/point — every headline bytes/point figure\n(Segment::bytes_per_point, StorageEstimate.bytes_per_point /\ntotal_bytes_per_point, the bench HTML val B/pt) reports the codec actually\nwritten to disk, not a fixed-width estimate (bench schema v10). The naive\nlen * width figure is retained beside it (Segment::logical_value_bytes,\nStorageEstimate.estimated_value_bytes) as the uncompressed baseline, so\nrealized / logical reads directly as the storage compression ratio.
    • \n
\n
Segment store (.weftseg + weftdb::SegmentStore)
\n
    \n
  • Sealed, checksummed segments — a Segment binds a typed value column and a\ndelta-of-delta timestamp column with min/max timestamp/value stats, row/null\ncounts, and a format version. The on-disk frame is hand-rolled, versioned, and\nCRC-32-verified before parse, so a corrupt or truncated file fails fast.
    • \n
  • Multi-codec timestamps — the timestamp block writes its second-difference\nstream under whichever of five codecs is smallest — fixed-width bit-packing,\nper-value zig-zag varint, RLE, Gorilla variable-length, or per-block\nadaptive (dynamic) bit-packing — chosen by a self-describing selector byte the reader\ndispatches on, routed through the single source of truth so the reported codec name\nalways matches the bytes on disk. A regular series packs to a handful of bytes on disk\n(a 1000-point regular block stores in <20 bytes), so the bytes/point saving is\nrealized, not just estimated. The value block carries its own codec selector too,\nso a ScaledI64 column stores its mantissas under whichever of varint, fixed-width\nbit-packing, per-block adaptive (blocked) bit-packing, or per-block\nFrame-of-Reference (FOR) packing is smallest on disk.
    • \n
  • Order semantics — every segment records whether its timestamps are monotonic\n(time_sorted). Ingest can enforce order (require_sorted rejects an\nout-of-order batch rather than sealing it), and the per-aspect/store rollups carry\nan unsorted_segments count so out-of-order data is visible before it costs a\npoint-lookup scan.
    • \n
  • Order-signal read planner — a single-instant point lookup\n(SegmentStore::read_point) prunes the index to the segments spanning the instant\nand resolves each with its persisted time_sorted flag: a sorted segment is\nbinary-searched, an out-of-order one linear-scanned (the only sound search on\nunsorted timestamps). Both frame kinds resolve through the streaming point read\n(weftseg::read_segment_point / read_paged_segment_point, below) — no value-column\nmaterialization on a per-block codec, and a paged frame prunes pages on their indexed\nmin/max timestamp without decoding a column byte before touching the one surviving page.\nSegmentStore::read_points resolves a batch of instants in one pass — the index is pruned\nonce and each segment's timestamp column decoded once for the whole batch, so N instants\nsharing a segment cost one decode, not N (~54× faster for 64 instants on a 100k-row FOR\nframe than 64 single lookups — 67 µs vs 3.57 ms,\nweft-physical-type/benches/pointread.rs).
    • \n
  • Intra-segment reconciliation — SegmentStore::reconcile_segment/reconcile_aspect\nrewrite an out-of-order segment into a sorted one in place (stable sort by\ntimestamp, re-sealed at the same id, frame kind preserved), so it drops out of the\nunsorted_segments count and its point lookups binary-search.
    • \n
  • Threshold-triggered reconciliation — the unsorted_segments backlog drives a\nQuestDB-style trigger so the rewrite is paid only once out-of-order data is worth\ncompacting, never on every late row: reconcile_aspect_if_unsorted_exceeds gates a\nsingle-aspect pass on the backlog, reconcile_all_over_threshold sweeps every\ndeclared aspect over the threshold, and a background timer daemon\n(WEFT_RECONCILE_INTERVAL_SECS / WEFT_RECONCILE_THRESHOLD) runs the sweep\nunattended. A threshold of 0 clamps to 1 (any out-of-order segment). Passes and the\nsegments they rewrite are counted in weft_reconcile_* metrics.
    • \n
  • Hot/cold reconciliation — reconcile_aspect_hot_cold/reconcile_all_hot_cold\nreconcile every cold (sealed) out-of-order segment on each pass but defer the\nhot tail (the most-recently-sealed segment) until the backlog reaches the\nthreshold, so the actively-appended segment is not rewritten on every tick —\nQuestDB's "squash non-active partitions each commit, defer the active one" policy.\nEnabled on the daemon with WEFT_RECONCILE_HOT_COLD and on the reconcile endpoints\nwith ?hot_cold=true.
    • \n
  • Cross-segment overlap signal — a sorted segment whose time window a\nlater-sealed segment re-enters (late data landing in an already-covered window) is\nnot counted by unsorted_segments; SegmentIndex::overlapping_count reports these\ntime-overlapping segments as a distinct overlapping_segments order-health\ncount in the per-aspect and store-wide rollups.
    • \n
  • Cross-segment overlap merge — SegmentStore::reconcile_overlaps/reconcile_all_overlaps\ncollapse each connected group of time-overlapping segments into one time-sorted\nsegment, folding members oldest→newest with newer-wins (last-writer-wins /\nupsert) dedup on shared timestamps — the same answer a point lookup already gives\nacross overlaps, and QuestDB's DEDUP UPSERT "last write wins on the designated\ntimestamp" semantics. Drives overlapping_segments to zero; exposed on the reconcile\nendpoints with ?overlaps=true and as a background daemon sweep\n(WEFT_RECONCILE_OVERLAPS).
    • \n
  • Split-not-rewrite merge — SegmentStore::split_segment carves a sorted segment\nat a boundary into a cold prefix (kept id) + hot suffix (new id), and\nreconcile_overlaps_with_policy consults a size-based SplitPolicy: when an overlap\ncomponent's cold prefix clears a byte floor and outweighs its hot suffix, only the\nsuffix is merged and the large cold prefix is left untouched, so later late arrivals\nre-entering the hot window never rewrite it — QuestDB's partition-split write-amp\nbound. The default reconcile_overlaps keeps QuestDB's 50 MiB floor (small components\nfull-rewrite); a floor is selectable with ?split_min_bytes= on the per-aspect and\nstore-wide reconcile endpoints and WEFT_RECONCILE_SPLIT_MIN_BYTES on the daemon.
    • \n
  • Squash — SegmentStore::squash_aspect/squash_aspect_if_exceeds/\nsquash_all_over_threshold fold an aspect's segments back into one (newer-wins),\nbounding the fragmentation repeated split carve-offs create — QuestDB's\nmax.splits squash trigger. Exposed as POST …/{aspect}/squash?max_segments= and on\nthe daemon with WEFT_RECONCILE_MAX_SPLITS (squash aspects over the cap each tick,\nafter the overlap merge that accumulates the splits).
    • \n
  • Null/quality column — a per-row presence bitmap (zero bytes for fully dense\ncolumns) makes gaps real: present values are stored densely and reconstructed\nas None on read.
    • \n
  • Paged segments — rows partition into fixed-height pages, each independently\nencoded with its own stats, behind a per-page index table: a reader prunes\npages on min/max time/value without touching a column byte and seeks\nstraight to the pages it needs.
    • \n
  • Data skipping — segment-level and page-level pruning by time, by value\nband, and by quality (segments/pages that overlap a window but hold only\nnulls there are skipped). A range read over a regular (constant-stride)\nblock-coded segment goes further — read_segment_range computes the row\nwindow in closed form and unpacks only the present values inside it, so a\nselective range decodes ~window values, not the whole segment (~47× faster\nfor a 100-row window over a 100k-row FOR frame — 58 µs vs 2.73 ms,\nweft-physical-type/benches/pointread.rs).
    • \n
  • Control plane — SegmentIndexStore persists one descriptor per sealed\nsegment in libSQL and answers range queries with SQL pruning;\nCatalogStore/AspectCatalog register the database → subject → aspect\nhierarchy with declared schemas (declare once, seal by lookup — an undeclared\naspect is refused, not guessed); per-aspect metadata.db rollups make\naspect-wide stats (including realized bytes/point) an O(1) read, always\nre-derivable from the segment index.
    • \n
\n
Arrow & Parquet interchange (weft-arrow, weft-arrow-store)
\n
    \n
  • Lossless Arrow batches — sealed segments and stored reads convert to/from\nApache Arrow RecordBatches: a lossless text form, plus a typed fast path\n(F64 → Float64, F32 → Float32, scaled integers → exact Decimal128).\nPaged segments stream one batch per page.
    • \n
  • Wire formats — Arrow IPC stream bytes and Parquet files (embedding\nthe Arrow schema so WeftDB's time-unit/encoding metadata survives), both\ndirections: export a stored range, or ingest a Parquet file into a declared\naspect under the full no-silent-downcast guarantee.
    • \n
  • Lean core preserved — the arrow-*/parquet dependency tree lives only in\nthese leaf crates; splimes/weftdb/weft-physical-type stay arrow-free.
    • \n
\n
\n

The Analytics Pipeline

\n

The weft-orchestration crate provides a fluent Pipeline API. Each aspect\nhas exactly one pipeline, persisted to its pipeline.db, so runs are incremental\nand conflict-free. A pipeline can host multiple dictionaries and multiple detectors.

\n
 Measurements ─▶ Batches ─▶ Processed Batches\n                                   │\n                                   ▼\n                          Dictionaries (1..N)        Event Detectors (1..N)\n                          extract patterns           peaks · valleys ·\n                                   │                  thresholds · monthly · …\n                                   ▼                          │\n                               Patterns ───── correlate ──────┤\n                                                              ▼\n                                  Events ─▶ Correlations ─▶ Signals\n                                                              │\n                                                              ▼\n                                          query_probability(event, time)

Built-in event detectors (in weft_orchestration::detectors):

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
DetectorDescription
detect_monthly_increaseMonths where the value rises ≥ a threshold start-to-end.
detect_peaks / detect_all_peaksGlobal / all local maxima.
detect_valleys / detect_all_valleysGlobal / all local minima.
detect_threshold_crossing_up / _downUpward / downward threshold crossings.
detect_drawdownSignificant value drops.
\n

You can also register custom detectors with the event_detector_fn! macro, and\nrun pipelines across many aspects in parallel with run_all_pipelines /\nrun_subject_pipelines.

\n
\n

Dataset Compression

\n

Long-running deployments accumulate huge volumes of raw measurements. The\ncompression module (weftdb::compression) reduces storage while preserving the\nshape of the data, configured per-aspect via CompressionConfig:

\n
    \n
  • Time-based — recent data within a "pure" window (e.g. the last 7 years) stays\nuncompressed; older data is progressively compressed across configurable tiers\nwith Linear, Exponential, or Custom aggressiveness scaling.
    • \n
  • Size-based — compress until total storage fits within a target (e.g. 10 GB),\nhitting the oldest data hardest. Takes precedence over time-based.
    • \n
\n

Compression works by interpolating to a coarser resolution and applying\nslope-change simplification to drop redundant points. Newly inserted data inside a\ncompressed range marks a dirty region for targeted re-compression. Progress is\nreported live (phase, tier, aggressiveness, time range) — surfaced in the TUI.

\n
use weftdb::{CompressionConfig, TimeBasedCompressionConfig};\n\nlet config = CompressionConfig::time_based(\n    TimeBasedCompressionConfig::default_seven_years()\n);\nlet aspect = db.track_aspect(&subject_id, \"temperature\", &Resolution::Hours, Some(config)).await?;\nlet summary = aspect.compress(&db).await?;
\n

Benchmarking

\n

Weft-Bench (weft-bench) is WeftDB's reproducible, correctness-gated benchmark\nharness — both an internal engineering suite and a public, customer-runnable\ndiagnostic. Every number it emits ships with the dataset seed, the workload\nprofile, and a correctness verdict.

\n
\n

Policy: WeftDB benchmarks guide real engineering decisions, not synthetic wins.\nA latency number is only publishable when its correctness check passes.

\n
\n

What it does today:

\n
    \n
  • Seeded workload profiles — the flagship interpolation-heavy-irregular\nprofile generates an irregularly-spaced, gap-containing series\n(ChaCha8Rng, published seed → byte-for-byte reproducible). The underlying\nanalytic signal is shape-selectable (MultiSine | Sawtooth | Step |\nDampedSine) and doubles as the accuracy ground truth.
    • \n
  • Storage, read & aggregation workloads — beside the flagship interpolation\nrun, four parallel workloads exercise WeftDB's own hot paths, each with its own\ncorrectness gate and p50/p95/p99 latency: point_lookup (--point-lookup)\nseals a .weftseg segment and times the streaming point read\n(read_segment_point/read_segment_points, single or batch, single-block or\npaged via --pl-rows-per-page, regular closed-form vs irregular), gated against\nthe full-decode value — also over a real corpus with --pl-csv (1M real BTC closes:\na 128-instant batch at p50 2.43 ms single-block vs 0.90 ms paged at 8,192 rows/page,\nweft-bench --point-lookup --pl-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --pl-rows 1000000 --pl-queries 128 [--pl-rows-per-page 8192] --reps 50);\nrange_fetch (--range-fetch, real corpus via --rf-csv: 32 × 100-row windows over 1M\nreal BTC closes at p50 60.8 ms single-block vs 17.8 ms paged) times the windowed\nrange read; compression (--compression) reports realized bytes/point, the\nvalue-column compression ratio, and decode throughput, gated on an exact\nround-trip — over a seeded shape, or a real corpus with --comp-csv <FILE>\n(--csv-value-col, --csv-skip; values read as exact decimal text; --pl-csv does the same for point_lookup). On 1M real\nBTC/USD one-minute closes (rows 3M onward of btc_1min.csv) it measures 4.22 B/point\nrealized (scaled_for, exact), full-decode 20.8M points/sec (p50 52.7 ms, 5 reps),\nand an advisory decimal-exponent FOR footprint of 1.70 B/point\n(weft-bench --compression --comp-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --comp-rows 1000000 --reps 5); and downsample (--downsample) times WeftDB's canonical\nweft-reduce reduction into grid-aligned buckets with a\n--ds-aggs selector over min/max/avg/sum/first/last/p50…p99/twa/twa_linear/twa_bucket_end/sketch_p50…sketch_p99.\n--ds-parallel <N> reduces in N chunks via mergeable partial reductions (identical\nbuckets to serial, asserted by test) — measured 14.7× at 64 chunks on a quiet 16-core box\n(441.2 ms → 30.1 ms, 1,134,659 → 16,921,104 points/sec, --ds-aggs sketch_p99, 500k points,\n5 reps, correctness PASS); a re-run on 2026-10-08 at load average ~19 measured 327.3 → 48.5 ms\n(6.7×).\nThe real-corpus figures in this bullet come from btc_1min.csv, a local file not distributed\nwith the repository (see Ingest); point --pl-csv / --rf-csv / --comp-csv /\n--ds-csv at a CSV of your own to run the same workloads on real data.\nThe underlying point/range read speedups are quantified at the codec layer in\nweft-physical-type/benches/pointread.rs.
    • \n
  • Vendor-neutral adapters — every system is driven through the\nSystemAdapter trait: the WeftDB reference adapter (splimes' Interpolator on Backend::Auto),\na precision-aware portable linear baseline (fair-protocol class C), and a\nforward-fill/LOCF baseline (class B — the in-process mirror of\nFILL(previous) / locf()).
    • \n
  • Quality beside speed — reconstructions are scored against the analytic\nground truth: RMSE / MAE / max-error / bias. Honest finding, surfaced not\nhidden: on smooth multisine (and step) WeftDB's cubic leads on RMSE, but on\nsawtooth the portable linear baseline out-accuracies the cubic, which\novershoots sharp discontinuities.
    • \n
  • Fair-protocol statistics — p50/p95/p99 + min/max/mean/stddev with\nseeded bootstrap confidence intervals, and an end-to-end timing breakdown\nproving the latency distribution times only the adapter call, never setup.
    • \n
  • Storage estimate — each result carries a StorageEstimate (recommended\nencoding + value/timestamp/total bytes/point) computed by\nweft-physical-type over the actual columns. It also surfaces advisory\npotential-saving estimates for codecs not yet realized on disk: the best-of\nf64 value codec (Gorilla / Chimp / Chimp128 XOR, advisory_best_f64_bytes /\n_codec) for a lossy F64 column, and the Sprintz FIRE forecaster's\nfootprint on the timestamp column (advisory_fire_timestamp_bytes), and the\ndecimal-exponent FOR value codec (advisory_dfor_value_bytes, schema v17,\npresent only when it beats the realized codec — e.g. a ScaledI64 column whose\nscale is forced by a few high-precision values; real BTC closes measure 13.58 vs\n33.74 bits/value in\nweft-physical-type/benches/alp_vs_f64_codecs.rs),\nplus two timestamp-column what-ifs (schema v18), each present only when it beats the\nrealized codec: advisory_common_multiple_timestamp_bytes (the GCD of the deltas\nfactored out — millisecond-precise instants stored in microseconds measure 20 → 10\nbits/value in\nweft-physical-type/benches/timestamp_vs_pco.rs)\nand advisory_delta_for_timestamp_bytes (first-order deltas under per-block FOR, for\nindependent random intervals: 20.0 → 19.4 bits/value)\n— each a what-if number the adopt-or-drop decision reads, never a realized\nheadline claim. The f64 codecs and FIRE live behind weft-physical-type's\nexperimental-codecs feature, which only weft-bench enables.
    • \n
  • Reports — a BenchReport JSON artifact (run metadata + a best-effort\nhardware probe: CPU model, cores, RAM, and the kind/file system/mount of the disk under the working directory) under reports/json/, plus a\nself-contained HTML view (--html) with the most-accurate row highlighted.
    • \n
  • The engine the server runs — an interpolation run first calls\nsplimes::calibrate() once, as weft-server does at startup (skipping a\nCPU/software adapter the same way), so Backend::Auto uses rayon and the GPU\nwhere they are faster on this machine. The calibration, GPU and thresholds are\nprinted and recorded in the report's metadata.engine (schema v16);\n--no-gpu-calibrate skips it and records splimes' defaults. There is no\ncounterpart to the server's WEFT_GPU_CALIBRATE=force: on a software adapter the\nbench always skips calibration, so there it cannot reproduce a server started with\nforce.
    • \n
  • ILP / TSBS input — a .lp / TSBS file drives the same harness, correctness\ngate, and reporting via the shared weft-line-protocol parser.
    • \n
\n
# Benchmark a line-protocol file, WeftDB vs the portable baselines, JSON + HTML report:\ncargo run -p weft-bench -- \\\n    --input data.lp --field usage --spline cubic --resolution minutes --compare --html\n\n# Seeded synthetic mode — the only mode with a known ground truth, so it also scores accuracy:\ncargo run -p weft-bench -- \\\n    --synthetic --points 300 --noise 0 --shape sawtooth \\\n    --spline cubic --resolution minutes --reps 5 --compare\n\n# Storage/read/aggregation workloads over WeftDB's own hot paths:\ncargo run -p weft-bench -- --point-lookup --pl-rows 100000 --pl-queries 128 --reps 50\ncargo run -p weft-bench -- --range-fetch --rf-window 100 --rf-windows 32 --reps 30\ncargo run -p weft-bench -- --compression --comp-shape clustered --reps 20\ncargo run --release -p weft-bench -- --compression --comp-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --comp-rows 1000000 --reps 5\ncargo run -p weft-bench -- --downsample --ds-bucket m --ds-aggs min,max,p99,twa --reps 20

The process exits non-zero when any result's correctness gate fails, so scripted\ncallers can gate on it. External-engine competitor adapters (DuckDB, ClickHouse,\nInfluxDB 3, QuestDB, TimescaleDB) are the next tracked step in\nROADMAP.md.

\n
\n

Development

\n
# Run the full test suite:\ncargo test\n\n# Skip long-running tests:\nSKIP_SLOW_TESTS=1 cargo test          # PowerShell: $env:SKIP_SLOW_TESTS=1; cargo test\n\n# Test a single crate:\ncargo test -p weftdb

The workspace ships extensive Criterion benchmark\nsuites alongside the Weft-Bench harness (see\nBenchmark Harness):

\n\n\n\n\n\n\n\n\n\n\n\n
CrateBenchmarks
weftdbinterpolation_benchmarks, integration_interpolation_benchmarks, optimized_interpolation_benchmarks, strategy_selection_benchmarks, new_api_benchmarks, cache_performance_benchmarks
\n
cargo bench -p weftdb --bench cache_performance_benchmarks

These benches create their databases in a temporary data dir of their own, which they\nremove when they finish, never in ~/.weftdb/data. That dir is under the system temp\ndir, often a tmpfs; to bench another filesystem, set TEST_DATA_DIR to a directory on\nit (the benches remove the databases they create there, not the directory).

\n

HTML reports are generated under target/criterion/. The interpolation engine's own\nbenchmarks live in the splimes repository.

\n

Formatting uses nightly-only rustfmt options (imports_granularity,\ngroup_imports), so the style gate runs on nightly even though the crates\nthemselves build on stable:

\n
cargo +nightly fmt --all
\n

Project layout

\n
WeftDB/\n├── Cargo.toml                  # workspace manifest (members + shared deps)\n├── weftdb/                     # time-series database (Turso/libSQL control plane + segment store)\n│   └── src/types/\n│       ├── database/           # connection, config, inputs, outputs, pipeline\n│       ├── compression/        # tiered dataset compression\n│       ├── batches/            # batching & analysis\n│       └── …                   # patterns, events, correlations, signals, …\n├── weft-orchestration/         # pipeline orchestration & event detectors\n├── weft-physical-type/         # physical encodings, schemas, timestamp codecs, .weftseg\n├── weft-arrow/                 # Arrow/Parquet interchange for segments\n├── weft-arrow-store/           # Arrow/Parquet bridge over the segment store\n├── weft-line-protocol/         # InfluxDB Line Protocol parser (shared dialect)\n├── weft-reduce/                # downsampling reductions over a bucket grid\n├── weft-server/                # axum HTTP API server\n├── weft-bench/                 # benchmark harness (adapters, profiles, reports)\n└── weft-tui/                   # terminal UI binary

The complete work queue and design constraints live in\nROADMAP.md — the single source of truth.

\n
\n

Crates

\n

WeftDB is a Rust Cargo workspace, from a low-level numerical engine up to an HTTP\nserver and an interactive application:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
CrateRole
splimes (own repo, from crates.io)Spline interpolation engine — Linear / Quadratic / Cubic / Polynomial methods on the CPU (one thread or rayon) or the GPU, chosen by grid size, with raw / interpolated / extrapolated provenance on every point.
weftdbTime-series database: Turso (libSQL) control plane (catalog, metadata, segment index; MVCC concurrent writes) + WeftDB's own typed columnar .weftseg segment store on the measurement hot path, plus the pattern-recognition types and tiered dataset compression.
weft-orchestrationHigh-level pipeline that chains batching → pattern extraction → event detection → correlation → signal generation, with built-in detectors and parallel execution.
weft-physical-typeVendor-neutral physical type system — schema-declared numeric encodings with explicit exactness, timestamp codecs, and the .weftseg columnar segment format (single-block and paged).
weft-arrow / weft-arrow-storeApache Arrow / Parquet interchange for sealed segments and stored reads, kept in leaf crates so the arrow-* dependency tree never reaches the hot-path core.
weft-line-protocolDependency-free InfluxDB Line Protocol parser shared by the server and the benchmark harness.
weft-reduceVendor-neutral downsampling reductions, computable over parts and merged (reduce_partial/PartialReduction, exact for every reduction) — min/max/avg/sum/first/last + nearest-rank p50/p90/p95/p99 + three time-weighted averages (LOCF, linear/trapezoidal, LOCF-to-bucket-end) + mergeable bounded-error sketch_p* percentiles, over an epoch-aligned bucket grid, computed in BigDecimal. A PartialReduction is serde-serializable (so a segment's partial can be persisted and merged later, in place of re-reading it) and re-bucketable to any coarser nesting resolution (rebucket/grids_nest). Shared by the HTTP downsample endpoint and the benchmark harness.
weft-serverBenchmark-grade axum HTTP API — interpolation, downsampling, storage ingest/query, catalog management, Prometheus metrics, live latency profiles.
weft-benchReproducible, correctness-gated benchmark harness — the roadmap's spine; both an internal suite and a customer-runnable diagnostic.
weft-tuiTerminal user interface (Ratatui + Crossterm) for creating databases, importing CSV data, browsing and plotting aspects, and running compression.
\n

The platform is designed for real-time and large-scale workloads: measurements\nare stored with BigDecimal logical precision,\ninterpolation scales from a handful of points to millions, from one CPU thread to the\nrayon pool and, where calibration measured it faster, the GPU, and the storage layer\nuses bulk transactions, cached connections, and typed columnar segments throughout.

\n
\n

Tuning tips

\n
    \n
  • Batch your writes. batch_capture_measurements uses bulk transactions and is\ndramatically faster than single inserts.
    • \n
  • Connections are cached. The database layer reuses connections through a\nTTL/LRU cache (default: 50 connections, 30-minute TTL) and starts MVCC\nBEGIN CONCURRENT transactions automatically, with retry + backoff on\ntransient failures.
    • \n
  • Calibrate the backends once. weft-server does it in the background at startup; a program\nembedding the libraries calls splimes::calibrate() (or splimes::prewarm_gpu()\nplus splimes::set_auto_thresholds) once, off any latency-critical path. Without\nit, interpolation never uses the GPU.
    • \n
  • Pick sensible batch sizes for pipelines (typically 24–100 for hourly data).
    • \n
  • Let the engine choose. Backend::Auto (behind analyze_range and every\nendpoint) already selects the backend by grid size — overriding is rarely necessary.
    • \n
  • Pipeline pattern loading is memory-aware: batch sizes adapt to available RAM to\navoid exhaustion on large datasets.
    • \n
\n
\n

Roadmap

\n

ROADMAP.md is the single source of truth for everything planned,\nin progress, and done — a phase-based [ ]/[x] checkbox queue organized around\nthe benchmark-led commercial thesis (north star: dollars per billion\ninterpolated output points at a p95 latency target). Shipped capabilities are\ndocumented here as features; every performance claim links to a benchmark\nartifact.

\n
\n

License

\n

Copyright (c) 2025-2026 Justin Icenhour.

\n

Licensed under either of

\n\n

at your option.

\n

Two crates include code adapted from Apache-2.0 projects — the Chimp codecs in\nweft-physical-type and the DDSketch quantile sketch in weft-reduce. Those portions\nstay under the Apache License 2.0 whichever option you choose. NOTICE lists\nthem, and each crate's THIRD-PARTY-NOTICES file carries the attribution and the\nlicense text.

\n

Contribution

\n

Unless you explicitly state otherwise, any contribution intentionally submitted for\ninclusion in the work by you, as defined in the Apache-2.0 license, shall be dual\nlicensed as above, without any additional terms or conditions.

\n

Every pull request also needs either a DCO sign-off on each commit or a signed CLA; see\nContribution terms.

\n", + "readmeHtml": "

WeftDB

\n

The time-series database for data that wasn't sampled on a clean grid.

\n

\"Release\"\n\"CI\"\n\"License:\n\"Rust\"\n\"Platform\"\n\"Status\"

\n

Most time-series databases assume your data arrives on a regular cadence. Real data\noften doesn't. Sensors drop out. Markets trade in bursts. Devices batch-upload after\nhours offline. When you ask those databases for a value between two samples, you get\na gap, a NULL, or a 0 — and the work of reconstructing the signal lands in your\napplication code.

\n

WeftDB puts that reconstruction in the database. Ask for any resolution you like\nand it returns a continuous series, interpolated with the spline method you choose,\nwhere every point is labelled raw, interpolated, or extrapolated — so a\nsynthetic value is never silently mistaken for an observed one.

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/interpolate \\\n  -H 'content-type: application/json' \\\n  -d '{\"spline\":\"linear\",\"resolution\":\"seconds\",\n       \"points\":[{\"timestamp\":\"1970-01-01T00:00:00Z\",\"value\":0.0},\n                 {\"timestamp\":\"1970-01-01T00:01:00Z\",\"value\":60.0}]}'

Two samples a minute apart come back as 61 labelled, per-second points — the two\nobserved endpoints marked raw, the 59 between them interpolated.

\n
\n

Contents

\n

Start here · Why WeftDB · Quickstart · Is WeftDB right for you? · Project status

\n

Use it · What you get · HTTP API · Configuration · Rust library · Terminal UI

\n

Understand it · Performance · How it works · Benchmarking · Development

\n
\n

Why WeftDB

\n

Interpolation is a first-class query, not post-processing. Resampling, gap\nfilling and upsampling run next to the data, over typed columnar segments, on one CPU\nthread, the rayon pool or — once calibrated — the GPU, chosen by grid size, not in a\nclient loop pulling raw rows across the wire.

\n

Precision is declared, never silently lost. Values are logically\nBigDecimal. Each aspect declares a physical encoding\n(F64, F32, ScaledI64, ScaledI128, Decimal128, BigDecimalText) and an\nerror bound. A value the encoding cannot represent within that bound is rejected,\nnot quietly rounded. If you are storing money or calibrated instrument readings, this\nis the difference between a database you can audit and one you can't.

\n

Synthetic data is always marked as synthetic. Every reconstructed point carries\nits provenance. Downstream code can filter, weight or refuse interpolated values on\npurpose instead of by accident.

\n

Claims come with benchmarks. Every performance number in this document links to\nthe harness that produced it, and the harness ships in the repo so you can re-run it\non your hardware. Where WeftDB doesn't win, this README says so.

\n
\n

Quickstart

\n

Prerequisites: Rust 1.95 or newer on stable. A GPU with a wgpu backend\n(Vulkan / Metal / DX12) is optional — without one the engine runs on the CPU\n(one thread or the rayon pool) automatically.

\n
cargo build --release

Prefer not to build from source? Each\nGitHub release, from v0.1.0 on,\ncarries pre-built weft-server, weft-tui and weft-bench archives for Linux (x86_64\nand aarch64), macOS (x86_64 and Apple silicon) and Windows (x86_64), and a SHA256SUMS\nfile to check them with sha256sum -c SHA256SUMS. The Linux binaries are built on\nUbuntu 24.04: weft-server and weft-bench need glibc 2.34 or newer, and weft-tui\nneeds glibc 2.39 or newer. The macOS and Windows binaries are not signed, so Gatekeeper\nand SmartScreen will warn before running them. Read the release's known limitations\n(v0.1.0's are in its\nchangelog)\nbefore you run it.

\n

1. Start the server

\n
WEFT_SEGMENT_STORE_ROOT=./store cargo run --release -p weft-server\n# weft-server v0.1.0 listening on http://127.0.0.1:8080

WEFT_SEGMENT_STORE_ROOT is what enables the persistent /storage endpoints;\nwithout it the compute endpoints still work statelessly.

\n

2. Interpolate without storing anything

\n

The compute endpoints take points in the request, so you can evaluate WeftDB\nbefore committing any data to it:

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/interpolate \\\n  -H 'content-type: application/json' \\\n  -d '{\"spline\":\"cubic\",\"resolution\":\"seconds\",\n       \"points\":[{\"timestamp\":\"1970-01-01T00:00:00Z\",\"value\":0.0},\n                 {\"timestamp\":\"1970-01-01T00:00:30Z\",\"value\":30.0},\n                 {\"timestamp\":\"1970-01-01T00:01:00Z\",\"value\":60.0}]}'

Already have InfluxDB Line Protocol? Point it at the ILP sibling instead:

\n
printf 'cpu,host=a load=0 1000000000\\ncpu,host=a load=60 1000000060\\n' | \\\n  curl -s -X POST \\\n    'http://127.0.0.1:8080/api/v1/interpolate/ilp?field=load&precision=s&spline=linear&resolution=seconds' \\\n    -H 'content-type: text/plain' --data-binary @-

3. Store a series and query it back

\n

Declare the aspect's schema once — this is where you choose precision:

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/storage/aspects \\\n  -H 'content-type: application/json' \\\n  -d '{\"name\":\"price\",\"physical_type\":\"scaled_i64\",\"scale\":2,\n       \"value_tolerance\":\"0\",\"timestamp_unit\":\"millis\"}'

scale: 2 with value_tolerance: "0" means exactly two decimal places, no rounding\npermitted — a third decimal is rejected at ingest rather than silently truncated.

\n
# ingest\ncurl -s -X POST http://127.0.0.1:8080/api/v1/storage/price/points \\\n  -H 'content-type: application/json' \\\n  -d '{\"points\":[{\"timestamp\":1000,\"value\":\"1.25\"},\n                 {\"timestamp\":2000,\"value\":\"2.50\"},\n                 {\"timestamp\":3000,\"value\":\"3.75\"}]}'\n\n# read the range back (start/end are required, in the aspect's declared unit)\ncurl -s 'http://127.0.0.1:8080/api/v1/storage/price/points?start=0&end=9999&limit=10'\n\n# read one instant\ncurl -s 'http://127.0.0.1:8080/api/v1/storage/price/at?t=2000'\n\n# bucket it\ncurl -s 'http://127.0.0.1:8080/api/v1/storage/price/downsample?resolution=seconds&agg=avg,max,p95'

Every read surface also speaks CSV, Arrow IPC and Parquet — append .csv,\n.arrow or .parquet, or use the …/range.parquet variants, and pipe straight into\npandas, Polars or DuckDB.

\n
\n

Is WeftDB right for you?

\n

Being honest about this saves you an evaluation.

\n

A good fit

\n
    \n
  • Irregular or gappy series where you need values between the samples —\nsensor networks with dropouts, market data, batch-uploaded device telemetry.
    • \n
  • Resampling and gap filling as a hot path, run often enough that doing it in\napplication code is costing you real latency or CPU.
    • \n
  • Values where precision is not negotiable — prices, billing quantities,\ncalibrated scientific readings — and a silent f64 round-trip is unacceptable.
    • \n
  • Workloads that care which points are real, because a synthetic value would\notherwise contaminate an alert, a model, or an invoice.
    • \n
  • Point lookups and narrow windows over large columnar segments, which WeftDB\noptimises specifically (see Performance).
    • \n
\n

A poor fit — today

\n
    \n
  • You need a mature, hardened production database. WeftDB has no TLS, no\nauthentication, and no RBAC yet. Do not expose it to an untrusted network.
    • \n
  • Generic OLAP over wide tables. WeftDB is not competing with ClickHouse or\nDuckDB on broad analytical scans, and doesn't claim to.
    • \n
  • Regular, evenly-sampled data with no gaps. If interpolation isn't your\nproblem, WeftDB's core advantage doesn't apply and a conventional TSDB is the\nsimpler choice.
    • \n
  • Per-measurement tags / labels. Not implemented yet, so tag-based filtering\nand multi-dimensional grouping aren't available.
    • \n
  • High-cardinality dimensional slicing in the Prometheus sense — that's not\nthe shape of WeftDB's data model.
    • \n
  • Managed hosting. There is none; you run it yourself.
    • \n
\n
\n

Project status

\n

Pre-beta. Expect sharp edges. WeftDB is an engineering-grade codebase — over\n1,000 tests pass, the storage format is CRC-checksummed and versioned, and the\nbenchmarks are real and reproducible — but it has not been hardened for production\ndeployment.

\n

v0.1.0 is the pre-beta baseline release: pre-built binaries on the\nGitHub release, and no\ncrates on crates.io. Later releases test their upgrade, rollback and compatibility\nagainst it. Its\nknown limitations\ninclude no\nauthentication or TLS, incomplete crash consistency, and a store layout that the next\nrelease upgrades one way.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AreaState
Storage engine, interpolation, HTTP API, Arrow/ParquetWorking, tested, benchmarked
Backup & restore (control plane)Working, with verification and a rehearsal drill
Observability — Prometheus metrics, OpenTelemetry tracesWorking
Crash consistency — power-loss durability, crash-safe maintenanceIn progress (design)
Security — TLS, auth, RBAC, audit logsNot started
Pre-built binaries (Linux, macOS, Windows)Released: v0.1.0, on GitHub
Library crates on crates.ioNot published yet
Packaging — Docker, HelmNot started
Client SDKs (Python, TypeScript)Not started
Per-measurement tagsNot started
\n

The full work queue lives in ROADMAP.md. It is the single source of\ntruth for status and priorities, and it records negative results alongside wins.

\n
\n

What you get

\n

Query

\n
    \n
  • Interpolation on read — reconstruct any series onto any grid from nanoseconds\nto years, with Linear / Quadratic / Cubic / Polynomial splines. The engine picks\nsingle-threaded CPU, parallel (Rayon) or — once calibrated — GPU by grid size, and\ndegrades the method gracefully (Cubic → Quadratic → Linear) when data is too sparse\nto support it.
    • \n
  • Provenance on every point — raw, interpolated or extrapolated.
    • \n
  • Downsampling into epoch-aligned buckets: min/max/avg/sum/first/last,\nexact nearest-rank p50/p90/p95/p99, three time-weighted averages (LOCF\nfor change-only sensors, trapezoidal for continuous signals, and an LOCF variant\nthat carries the last sample to the bucket's end), plus bounded-error\nsketch_p* percentiles that are mergeable and cost a fixed amount of memory per\nbucket however many samples land in it.
    • \n
  • Point and range reads that skip work: a point lookup on a regular column is\nresolved in closed form without materialising timestamps at all.
    • \n
\n

Store

\n
    \n
  • Typed columnar segments (.weftseg) — written once, never rewritten: maintenance\n(reconcile, split, overlap merge, squash, compaction) writes its output to new frames,\nfsyncs them, and swaps them for its inputs in one control-plane transaction, so a\ncrash leaves either the inputs or the outputs, and a replaced frame is deleted only\nonce no read can still be using it. CRC-checksummed, page-indexed, with per-segment\nand per-page min/max statistics for data skipping and a null/quality column.
    • \n
  • Compression that adapts per column — timestamps pick among delta-of-delta,\nRLE, Gorilla, fixed and per-block bit-packing; values pick among varint,\nbit-packing, per-block adaptive packing, frame-of-reference and an opt-in delta\ncascade. The codec each column actually used is reported, and bytes/point is the\nrealized figure, not an estimate.
    • \n
  • Late and out-of-order data — detected, counted, and reconciled within a segment or\nacross segments with last-writer-wins semantics, on demand or by background sweep.
    • \n
  • Backup and restore of the control plane, with two verification modes (one safe\nunder concurrent writes) and a non-destructive restore drill so you can answer\n"is my backup actually restorable?" without risking the live store.
    • \n
\n

Integrate

\n
    \n
  • HTTP API with JSON, CSV, Arrow IPC, Parquet and InfluxDB Line Protocol in and\nout — so TSBS/Influx/QuestDB-shaped writers work unchanged, and results land\ndirectly in Arrow-native tooling.
    • \n
  • Prometheus /metrics with latency histograms, plus a live p50/p95/p99 profile\nendpoint and OpenTelemetry trace export.
    • \n
  • Rust libraries — use the engine, the storage layer or the reductions directly,\nwithout the server.
    • \n
  • Terminal UI for importing CSV, browsing and plotting aspects, and running\ncompression interactively.
    • \n
\n
\n

Performance

\n

Every number below comes from a benchmark in this repo, named so you can re-run it. The\nexception is any figure measured on the real BTC corpus: that input is a local file the\nrepository does not ship (see Ingest), so those figures cannot be reproduced from a\nclone. All were measured on one developer workstation — treat them as shape, not as a\nspecification for your hardware.

\n

Reads WeftDB optimises specifically

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OperationWeftDBNaive full decodeGain
Point lookup, 100k-row segment56 µs2.60 ms~46×
Point lookup, paged frame33 µs279 µs~8.5×
Batch of 64 instants (one pass vs 64 single lookups)67 µs3.57 ms~54×
100-row window over 100k rows58 µs2.73 ms~47×
\n

Source: weft-physical-type/benches/pointread.rs,\nre-measured 2026-10-08 after the word-wise bit reads and the slicing-by-16 frame CRC.\nThe wins come from skipping the value column entirely until a specific row is needed,\nand from resolving regular timestamp columns in closed form.

\n

Ingest

\n

Sealing 1M rows of a real 1-minute financial series into a columnar segment:\n4.57 s — 218,963 rows/sec at 4.22 bytes/point.

\n

That corpus (database/datasets/btc_1min.csv, a BTC/USD one-minute series) is a local,\ngitignored file, not distributed with this repository and not required to build or test —\nthe suite skips the tests that use it when it is absent, and so do the benches that load it by\ndefault. Every "real BTC" figure in this README comes from it, so none of them can be reproduced\nfrom a clone. Whether those numbers are cleared for publication or retired is an open item with\ncounsel before 1.0 (1.0 plan, Appendix B).\nThe ingest figure is quoted anyway because a synthetic generator materially flatters the\nresult: on the same workload the generated corpus reports 1.71 bytes/point against the real\n4.22, roughly 2.5× too optimistic. Storage numbers measured on generated data are not reported\nas results.

\n

Reductions

\n

At 500k two-decimal points, hour buckets, correctness-gated\n(weft-bench --downsample; re-measured 2026-10-09 on the two-decimal generator, full\ntable under Compute endpoints): avg takes 4.5 ms (118M points/sec);\ntime-weighted average costs about 5.7× that. Exact p99 takes 23.5 ms and the approximate\nsketch_p99 about the same (18.8 ms), on real BTC closes too; what the sketch buys is bounded\nmemory per bucket and mergeability, within its 1% bound.

\n

Where it doesn't pay off

\n

A deliberately-kept honest result: the bit-sliced (transposed) value codec once\nmeasured ~5.7× faster than the linear bit-unpack at the kernel level, but only because\nthe linear decoder tested one bit at a time. Once that decoder moved to word-wise reads,\nthe linear layout unpacked faster (1.09 vs 1.95 ms per 1 Mi values,\nbenches/bitunpack.rs). End to end, the\nbit-sliced layout costs +0.08% bytes and reads no faster than the linear codec\n(benches/transposed_read.rs), so it\nships off by default (and is now behind the bitsliced-codec build feature, pending\npatent review). A kernel speedup measured against a weak baseline is not a result, and\nthis README would rather say so than quote the 5.7×.

\n

Exact decimals are not free yet either. Hourly avg over 1M real BTC closes (the local corpus\ndescribed under Ingest, not in the repository), all three paths proven to produce the\nsame buckets:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
pathtimevs f64
f64 loop (lossy baseline)2.06 ms1×
weft_reduce::reduce_scaled (exact, on the stored ScaledI64 mantissas)4.26 ms~2.1×
weft_reduce::reduce (exact, BigDecimal points; integer arithmetic inside since 2026-10-09)21.0 ms~10×
\n

(Re-measured 2026-10-11 on a heavily loaded box (load average ~200, not pinned), each ratio taken\nagainst the f64 loop in the same criterion run. reduce_scaled ran 4.26 and 4.61 ms against f64\n2.06 and 2.07 ms, i.e. 2.1–2.2×; in the same conditions before that day's change it ran 6.50 and\n7.78 ms against 2.08 and 2.54 ms, ~3.1×. The change: the integer accumulator closes time-ordered\nbuckets into a Vec instead of inserting each into a map, builds a bucket's sum decimal only when\nsum is asked for, and sizes the output once. On 2026-10-09 reduce_scaled measured 8.99 ms\n(criterion pinned to one core, median of nine alternating runs) and on 2026-10-08 13.52 ms. reduce\nran 24.1 ms (f64 2.08 ms) before the same change reached its points path, 21.0 ms (f64 2.09 ms)\nafter; it was 52.6–53.0 ms before it took the integer path for points that fit, and 60.04 ms on\n2026-10-08.)

\n

QuestDB documents ~2× for its DECIMAL; the\ninteger path is now about level with that on this reduction (a ratio against each engine's own\nDOUBLE/f64, not a cross-engine timing), while the BigDecimal-points path still costs ~10×. Before this work the shipped path measured 100.65 ms\n(~43×). Three changes closed most of the gap: integer\naccumulation; computing each bucket's avg by integer long division that reproduces\nbigdecimal's quotient digit for digit; and, for an average that never terminates (most of\nthem), producing its 100 digits with one BigUint multiplication and one division by the\ncount instead of a digit loop. Per sealed 1M-row segment (decode + six reductions), the\nstored-range downsample went from 127.6 ms to 59.7 ms. The server's GET /api/v1/storage/{aspect}/downsample takes the integer path for one-scale ScaledI64 segments for\nevery reduction (exact percentiles and time-weighted averages collect their samples from the stored\nmantissas, ~1.4× faster per segment for avg,p99,twa than before, 2026-10-09)\n(weft-reduce/benches/decimal_tax.rs, load average ~11).

\n

Three more results from 2026-10-09 where the obvious answer was wrong:

\n
    \n
  • The GPU is not the fastest way to interpolate stored data. Reconstructing ~1M one-second\npoints from 100k stored knots end to end (SegmentStore::interpolate_range_f64) takes 14.1–16.3\nms on rayon and 17.0–29.8 ms on the RTX 4070 Ti SUPER; with BigDecimal output both spend most\nof their time building a decimal per point\n(weftdb/benches/interpolate_range.rs).
    • \n
  • A spline gap fill is less accurate than a straight line on prices. On real BTC closes a\ncubic fill's MAE is $55.34 against linear's $49.27 (hourly buckets); the spline wins only on a\nsmooth signal (weft-bench --gap-fill --gf-fill cubic, see Benchmarking).
    • \n
  • The approximate percentile is not faster than the exact one. sketch_p99 once measured\n~2.6× faster than exact p99, but only on the old generator's ~50-digit values; on real-shaped\ndecimals the two run level (see Compute endpoints).
    • \n
\n
\n

HTTP API

\n

weft-server is the axum HTTP surface — Weft-Bench, TSBS-style harnesses, Grafana\nand SDKs all drive the engine through it. It carries no vendor-specific dependencies.

\n

See the Quickstart to start it.

\n

Bind address defaults to 127.0.0.1:8080 (WEFT_SERVER_ADDR overrides). Setting\nWEFT_SEGMENT_STORE_ROOT opens a Storage v2 segment store and enables the\n/storage endpoints (without it they answer 503, and GET /ready reports the\nsegment_store dependency). A store root belongs to one server at a time: the server\nholds the root's LOCK file while it runs, and a second server pointed at the same\nroot exits at startup with an error naming the first one's pid and host.

\n

The root's STORE_FORMAT file records the store's layout and the oldest layouts a\nWeftDB must know to read and to write it (plain JSON; see weftdb::StoreFormat), and\nsegment_index.db's store_meta table keeps a copy. The server reads the marker before\nit opens any database (a root without one, the copy, before it writes anything), and\nrefuses a store a newer WeftDB wrote (IncompatibleLayout) without changing anything\nbut LOCK and LOCK.holder. A store written before the marker existed is layout 1 and\nis upgraded in place on first open, through the registered migrations\n(store_migrations in segment_index.db lists those applied); its frames are not\nrewritten. A layout-1 store whose index records a frame outside segments/ (possible\nonly through the aspect-name traversal of earlier builds) is refused with\nUnsafeLegacyPath, naming the aspects, before any migration applies or a marker is\nwritten, and nothing is moved: drop or re-seal those aspects with the build that wrote\nthem first.

\n

While bound to a loopback address, the server only answers requests addressed to a\nloopback host: Host must be localhost, an address in 127.0.0.0/8 or [::1]\n(anything else is a 421), and a state-changing request whose Origin is not a\nloopback origin is a 403. That keeps web pages from non-loopback origins, open in a\nlocal browser, from driving the server; a page served by another local web server (any\nloopback origin, on any port) is still trusted until authentication lands. Clients that\nsend no Origin, like curl, are unaffected. /health and /ready follow the same\nrule, so a health probe must send a loopback Host such as localhost or 127.0.0.1,\nas every HTTP/1.1 client does; an HTTP/1.0 probe that sends no Host gets a 421, so\nconfigure it to send one or set WEFT_ALLOW_ANY_HOST=1. Behind a local reverse\nproxy that forwards a different Host, set WEFT_ALLOW_ANY_HOST=1; a proxy that keeps\nHost: 127.0.0.1 but forwards the Origin of a browser UI served from a non-loopback\norigin needs it too, or that UI's state-changing requests get a 403. A non-loopback\nbind is not guarded: there is no authentication yet, so keep such a server off\nuntrusted networks.

\n

Service endpoints

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Method & pathPurpose
GET /healthLiveness.
GET /readyReadiness, including the segment-store dependency check. With a store, poisoned / restart_required report that a segment-index COMMIT failed in a way that may still have committed, so the store refuses writes until the server restarts (reads keep working, and ready stays true); poison_reason says which transaction and error.
GET /metricsPrometheus text exposition — request/error/output counters, ingest counters (weft_ingest_*), and latency histograms for the compute and storage-ingest paths.
GET /debug/profile/currentLive p50/p95/p99 latency snapshot per instrumented path.
\n

Compute endpoints

\n

The flagship interpolation-on-read path and its reduction counterpart. Each\naccepts JSON points or an InfluxDB-Line-Protocol body, and each returns JSON,\nCSV, Arrow IPC, or Parquet:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
EndpointPurpose
POST /api/v1/interpolateReconstruct an irregular series onto a regular grid (spline = linear | quadratic | cubic | polynomial; resolution = nanoseconds..years). Every output point carries a kind — raw / interpolated / extrapolated. A grid of more than 10,000,000 points (MAX_INTERPOLATE_OUTPUT_POINTS, or WEFT_MAX_INTERPOLATE_POINTS) is refused with 400 before anything is computed; split larger ranges across requests. The response's spline is the method that ran: with fewer distinct timestamps than the requested method needs (cubic 4, quadratic 3, polynomial degree + 1) it is the simpler method splimes stepped down to.
POST /api/v1/interpolate/pointEvaluate the reconstructed signal at a single instant, labelled raw/interpolated/extrapolated.
POST /api/v1/downsampleReduce samples into epoch-grid-aligned buckets — min/max/avg/sum/first/last, nearest-rank percentiles p50/p90/p95/p99, the three time-weighted averages (twa, LOCF dwell-weighting, for change-only sensors; twa_linear, trapezoidal Σ½(vᵢ+vᵢ₊₁)Δtᵢ/ΣΔtᵢ, for irregularly-sampled continuous signals; twa_bucket_end, LOCF that additionally carries the bucket's last sample to its grid end — twa gives that sample no weight, so a sensor reporting 0 then 100 a minute into an hour bucket reads twa=0 but twa_bucket_end=98.33), and the approximate sketch_p50/p90/p95/p99 (see below); all reductions computed in BigDecimal by the shared weft-reduce crate. Aliases: median→p50, time_weighted_avg/twa_locf→twa, time_weighted_avg_linear→twa_linear, twa_locf_end→twa_bucket_end, sketch_median→sketch_p50. Only non-empty buckets are emitted.
POST /api/v1/{interpolate,downsample}/ilpThe same, fed an ILP text/plain body (the TSBS/InfluxDB/QuestDB wire format); field, precision (ns/us/ms/s), and the compute knobs are query parameters. interpolation= is accepted as an alias for spline= (the canonical spline wins if both are given).
POST /api/v1/{interpolate,downsample}/{csv,arrow,parquet} and …/ilp/{csv,arrow,parquet}The same computations with CSV (text/csv), Arrow IPC stream, or Parquet output — so a harness feeding line protocol pulls results in any of the four formats.
\n

What the reductions cost. Measured on the shipped harness (500k points at 1 s, 5 reps, hour\nbuckets, correctness PASS throughout — weft-bench --downsample --ds-points 500000 --ds-bucket h --ds-aggs <agg> --ds-decimals <2|full>; 2026-10-09, load average ~12–14, both generators back to\nback). Since 2026-10-09 the generator rounds its values to two decimal places, like the prices in the\nreal corpus (--ds-decimals, default 2). Before, it kept each float's exact binary expansion (~50\nsignificant digits, now --ds-decimals full), which made every BigDecimal operation expensive and\nthe published figures a pessimistic bound. The check: on 1M points, hourly avg,p99,twa, the\ntwo-decimal generator (60 s stride) reduces at 5.10M points/sec, real BTC/USD one-minute\ncloses at 4.49M points/sec (--ds-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000, the local corpus described under Ingest, not distributed with the repository),\nand the old generator at 0.61M points/sec, 7.3× slower than real data:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
reductionp50 (two decimals)throughputp50 (full, the pre-2026-10 generator)throughputnote
avg4.46 ms118,238,781 points/sec187.5 ms2,686,264 points/secthe streaming baseline — no bucket materialized
twa25.44 ms16,659,311 points/sec240.8 ms2,045,708 points/sec5.7× the streaming cost: dwell-weighting needs the time-ordered samples
twa_bucket_end24.40 ms17,340,324 points/sec241.4 ms2,045,536 points/secwithin noise of twa — one extra weight, effectively free
twa_linear68.76 ms6,919,376 points/sec313.5 ms1,578,596 points/sec2.7× twa: an extra BigDecimal add + divide per interval for the trapezoidal mean
p99 (exact)23.46 ms20,871,526 points/sec391.0 ms1,271,820 points/seccollects each bucket and selects the rank in linear time
sketch_p9918.79 ms25,871,740 points/sec280.1 ms1,785,661 points/secstreams into a DDSketch; see below
\n

(Re-measured 2026-10-09 after reduce started taking the integer path for values that fit\ni64 mantissas at a common scale; the two-decimal series does, the full one does not. The\nsame day's earlier figures, before that change: avg 14.4 ms, twa 49.0, twa_linear 99.3,\np99 31.5 (57.9 before the selection change), sketch_p99 30.3 (59.7 before the f64 fast\npath).)

\n

These time weft_reduce::reduce over BigDecimal points, the path POST /api/v1/downsample takes.\nSince 2026-10-09 reduce itself compares and sums on integers whenever every value fits an i64\nmantissa at the series' largest scale, with a result identical in value and representation\n(min, max, first, last and collected samples are the original values; each bucket's sum carries its\nown largest scale); on 1M real BTC closes, hourly avg went from 52.4 to 21.6–22.3 ms, sketch_p99\nfrom 71.0 to 42.3–42.9 ms (35.4–36.2 ms once the sketch also skips the logarithm for a value in the\nprevious value's bucket) and avg,p99,twa from 171.4 to 141.3–142.9 ms (two runs each). A stored\nScaledI64 aspect goes further, straight from the stored mantissas (reduce_partial_scaled, see\nWhere it doesn't pay off for the decimal tax), which --ds-scaled times on\nthe same series, gated on an identical result (10 reps, load average ~24): hourly avg over the 500k\ntwo-decimal points 0.58 ms, over 1M real BTC closes 6.4 ms; the six streaming reductions\n7.6 ms, sketch_p99 22.3 ms (17.9–18.9 ms with the bucket cache, three runs at load average ~33)\nand avg,p99,twa 109.0 ms on the real series (weft-bench --downsample --ds-scaled …).

\n

Since 2026-10-11 the time-weighted averages run on integers too: twa, twa_linear and\ntwa_bucket_end over samples whose mantissas fit i64 at scales up to 17 (mixed scales included,\nas decimal text has them) reproduce the BigDecimal loop's result digit for digit and scale for\nscale, and only the rest take the BigDecimal loop. On the same 1M real closes, hourly (17,477\nbuckets), against the morning's build in alternating runs at load average ~210, all PASS:\navg,p99,twa 150 → 55–58 ms on points and 119 → 38 ms on stored mantissas (--ds-scaled);\ntwa_linear 223–229 → 71–82 ms and 160–170 → 42–54 ms. The gain is mostly the per-bucket\n100-digit division, so it shrinks with fewer, larger buckets: on the 500k-point table series above\n(139 hourly buckets) twa and twa_bucket_end run level (22–25 ms both builds) and twa_linear\n78 → 53 ms.

\n

Exact vs sketch percentiles — which to ask for. The p50/p90/p95/p99 reductions are\nexact nearest-rank: they return an actual observed BigDecimal from the bucket, but they\nmaterialize and sort the whole bucket, and two buckets' results cannot be combined. The\nsketch_p50/p90/p95/p99 reductions instead stream every value into a\nDDSketch — a 1% relative-error bound\n(weft_reduce::SKETCH_ALPHA), bounded memory regardless of bucket size (values fold in as they\narrive; SKETCH_MAX_BINS caps the store absolutely), and an exactly mergeable structure, so a\np99 can be computed over a large or streaming bucket. Speed is not the reason to choose it. On the\nshipped harness (10 reps, correctness PASS, 2026-10-09) the two run level: p50 21.3 ms exact vs\n18.8 ms sketch on two-decimal values (500k points, hourly), and 43.4 ms vs 41.2 ms on 1M\nreal BTC closes (hourly), where values carry up to eight decimal places (weft-bench --downsample --ds-aggs p99 vs --ds-aggs sketch_p99, with --ds-csv for the real series). The exact percentile selects its rank in linear time over borrowed\nvalues, cloning only the result. The sketch converts a value with at most 15–16 significant digits\nand at most 22 decimal places to f64 by one division, which is provably the same f64 as\nbigdecimal's to_f64 (tested bit for bit); before that change sketch_p99 took 63.8 ms here. The\n~2.6× sketch advantage once quoted (327 vs 852 ms) was measured on the old generator's ~50-digit\nvalues, when the exact path also cloned and fully sorted every bucket.

\n

The approximation is declared, never silent — WeftDB's precision principle. The sketch shares the\nexact percentiles' nearest-rank convention (⌈q·n⌉), so sketch_p* and p* name the same\nsample at every bucket size and the sketch is always within the 1% bound of the exact answer —\nthe two are substitutable. (DDSketch's reference rank is ⌊q·(n-1)⌋, which on a small bucket picks\na different sample; WeftDB deliberately does not inherit that.) The exact percentiles remain the\ndefault and cost nothing on a small bucket, so prefer them there; reach for a sketch when a bucket\nis too large to materialize or the result must merge.

\n
curl -s -X POST http://127.0.0.1:8080/api/v1/interpolate \\\n  -H 'content-type: application/json' \\\n  -d '{\n        \"spline\": \"linear\",\n        \"resolution\": \"seconds\",\n        \"points\": [\n          { \"timestamp\": \"1970-01-01T00:00:00Z\", \"value\": 0.0 },\n          { \"timestamp\": \"1970-01-01T00:01:00Z\", \"value\": 60.0 }\n        ]\n      }'
printf 'cpu,host=a load=0 1000000000\\ncpu,host=a load=60 1000000060\\n' | \\\n  curl -s -X POST \\\n    'http://127.0.0.1:8080/api/v1/interpolate/ilp?field=load&precision=s&spline=linear&resolution=seconds' \\\n    -H 'content-type: text/plain' --data-binary @-

Malformed payloads, unknown tokens, fewer than two usable points, or a zero-span\nseries return 400 with an {"error": "..."} body.

\n

Numeric boundary: the compute endpoints' wire values are f64 — the\ndocumented transport boundary (the logical BigDecimal is narrowed only at the\nHTTP edge, and the columnar value columns are honestly typed Float64 — no false\nprecision). The storage endpoints below are lossless.

\n

Storage endpoints (Storage v2 over HTTP)

\n

Catalog management, ingest, and stored-range reads against the .weftseg segment\nstore. The no-silent-downcast guarantee holds end to end: a value unrepresentable\nunder the declared encoding/tolerance is rejected 400.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
EndpointPurpose
POST /api/v1/storage/aspectsDeclare an aspect's schema — physical encoding, value tolerance, timestamp unit. The name also names the aspect's files on disk, so it must be at most 160 bytes with no /, \\, control characters, bidirectional controls, line separators or invisible formatting characters (the zero-width joiner and non-joiner are allowed), leading ., trailing . or space, and must not be a Windows device name (CON, NUL, COM1, …, also with an extension or a : suffix, as in nul:x); a server running on Windows also refuses <, >, :, ", `
GET /api/v1/storage/aspects · …/{aspect}/schema · …/catalogList declared schemas; read one aspect's schema; report the store's (database, subject) scope + registered hierarchy.
POST /api/v1/storage/{aspect}/pointsIngest a JSON batch (dense or nullable; single-block or paged via rows_per_page). Set require_sorted to reject an out-of-order batch (400, naming the first backwards row) instead of sealing it. Every ingest response reports time_sorted (whether the sealed segment stored in monotonic order).
POST /api/v1/storage/{aspect}/{ilp,parquet,csv}Ingest an ILP payload, a Parquet file, or CSV rows into a declared aspect — all sealing through the same schema-enforced path, and all honouring require_sorted (query param) + reporting time_sorted.
GET /api/v1/storage/{aspect}/range · …/value-rangeStream a stored time window / value band as Arrow IPC (application/vnd.apache.arrow.stream).
GET …/range.parquet · …/value-range.parquetThe same windows as Parquet files.
GET …/range.csv · …/value-range.csvThe same windows as CSV (lossless decimal-text values; empty field = null).
GET /api/v1/storage/{aspect}/points · …/value-pointsLossless JSON reads with declarative pagination — offset/limit (+ take and 1-based page aliases), with total/count/offset in the body. For stable forward iteration the body also carries an opaque next_cursor while rows remain; a client re-issues with ?cursor=<token> (supersedes offset/page; malformed → 400) until it is absent.
GET /api/v1/storage/{aspect}/at?t=Single-instant point lookup: the present value at exactly t (lossless decimal text) or a found:false miss. An order-signal-driven read planner resolves each candidate segment with its persisted time_sorted flag — binary search on a sorted segment, linear scan only on an out-of-order one — after pruning the index to the files spanning t.
GET /api/v1/storage/{aspect}/at-multi?t=,,Batch point lookup: a comma-separated list of instants resolved in one pass (points:[{timestamp,value,found}] in query order). The index is pruned once by the batch's whole span and each segment's timestamp column decoded once for the whole batch, so N instants sharing a segment cost one decode, not N.
GET·POST /api/v1/storage/{aspect}/downsample?start=&end=&resolution=&agg=Stored-range downsample — the same reduction set as POST /api/v1/downsample (above), but over the aspect's persisted segments instead of a request-supplied series, so a client aggregates a large stored range without fetching it. Bounded memory: the index is pruned by time and each surviving segment folds into its own mergeable partial reduction, merged once — only one segment's rows are ever resident, so a range far larger than RAM reduces, and with a sketch_p* the per-bucket state is bounded too. Omitted bounds span all stored history. …/downsample.csv · …/downsample.arrow · …/downsample.parquet serve the identical reduction in the compute endpoints' columnar formats. Traced as a downsample.range stage span. The pruned segments are reduced concurrently: measured 4.69× faster than the sequential fold (16 segments over 200k rows, 39.88 ms → 8.50 ms; sketch_p99 49.49 ms → 9.42 ms, 5.25×) — see weftdb/benches/downsample_range.rs. A single pruned segment takes an inline fast path (concurrency there costs more than it buys). Materialized partials (opt-in): with WEFT_SEGMENT_PARTIAL_BASE set, each segment carries a .weftpart partial-reduction sidecar written at seal (a sealed segment is immutable, so its partial never goes stale), and a downsample of the bounded reductions (min/max/avg/sum/first/last + sketch_p*) merges the stored partials instead of decoding the value column — measured 3.2× faster (16 segments / 200k rows, 8.39 ms → 2.60 ms, non-overlapping CIs) and re-keyed to any coarser nesting resolution (minutes→hours→days) with no decode. Rollup tiers (opt-in): WEFT_SEGMENT_PARTIAL_TIERS materializes coarser tiers beside the base (e.g. hours,days), each re-keyed from the tier below, so a coarse query folds the coarsest matching tier instead of the whole fine base — the in-storage analogue of TimescaleDB's hierarchical continuous aggregates. Measured ~1.16× on a DAY query over a MINUTES base (16 segments / 200k rows, 7.30 ms → 6.30 ms, non-overlapping CIs). The win grows with the base-bucket count per segment, and that scaling is now measured rather than asserted: holding rows and segments fixed while widening the sample stride (so only the base buckets per segment change) gives 1.15× at ~208 base buckets → 1.27× at ~3125 → 1.38× at ~12500 (bench_tiered_span). Honest read: the win is real and climbs, but a ~60× increase in base buckets buys only 1.15×→1.38×, so the elided re-key is not the dominant cost and the tiers' extra .weftpart bytes should be weighed against it — see weftdb/benches/downsample_range.rs. A reconcile/split/squash regenerates the sidecar (tiers and all) so the acceleration survives a rewrite.
POST /api/v1/storage/{aspect}/reconcileReconciliation pass (mode in the response). Default (intra-segment): rewrite every out-of-order segment of the aspect into a time-sorted one in place. ?threshold=N gates it on unsorted_segments >= N (QuestDB-style split-count trigger). ?hot_cold=true reconciles cold segments but defers the hot tail until the backlog reaches the threshold. ?overlaps=true instead runs the cross-segment overlap merge (newer-wins) — reconciled is then the number of segments merged away, and ?split_min_bytes=N makes that merge split-not-rewrite (carve off a dominant cold prefix instead of rewriting the whole component). Returns triggered/reconciled/cold_reconciled/hot_reconciled and the post-pass unsorted_segments + overlapping_segments. Maintenance operations on one aspect take turns: while another one holds the aspect (a daemon pass, another request), the request waits up to 30 s, then answers 409 Conflict without touching the aspect; retry.
POST /api/v1/storage/{aspect}/squashSquash the aspect's segments into one (newer-wins), bounding split-path fragmentation. ?max_segments=N gates it (squash only when the count exceeds N). Returns triggered/removed/segment_count. Waits for a busy aspect and answers 409 as reconcile does.
POST /api/v1/storage/{aspect}/compact?target_rows=NSize-targeted compaction — coalesce the aspect's segments toward ~N rows per segment (leaving already-large segments untouched), holding fragmentation near the read-optimal size rather than folding to one (which squash does). Motivated by the downsample_range knee (one giant segment reads slower than several mid-sized ones). target_rows is required (absent → 400). Returns removed/segment_count. The manual counterpart of the WEFT_COMPACT_TARGET_ROWS daemon pass. Waits for a busy aspect and answers 409 as reconcile does.
POST /api/v1/storage/reconcileStore-wide reconciliation sweep across every declared aspect: threshold (default), ?hot_cold=true, or ?overlaps=true (+ ?split_min_bytes=N for split-not-rewrite). Returns mode, aspects_scanned / aspects_reconciled / segments_reconciled (+ cold/hot split), the post-sweep store-wide unsorted_segments + overlapping_segments, and failed: a [{aspect, error}] list of the aspects whose pass failed (empty on a clean sweep). A failing aspect (e.g. a torn or truncated frame) no longer aborts the sweep: every other aspect is still swept and the response is 200, so check failed to tell a partial sweep from a clean one (earlier versions returned 500 at the first failing aspect and left every aspect after it in name order unswept). Only an unreadable aspect list is still a 500. An aspect another maintenance operation holds is waited for (30 s in all, across the sweep); if some are still held then, the others are swept and the answer is 409 Conflict naming the held ones. (The background daemon never waits: it skips a busy aspect until its next tick.) The manual counterpart to the background reconcile daemon (WEFT_RECONCILE_INTERVAL_SECS / WEFT_RECONCILE_THRESHOLD / WEFT_RECONCILE_HOT_COLD / WEFT_RECONCILE_OVERLAPS / WEFT_RECONCILE_SPLIT_MIN_BYTES / WEFT_RECONCILE_MAX_SPLITS).
POST /api/v1/storage/backupOnline control-plane backup (Phase 7.4) — snapshot the store's four control-plane DBs (segment_index/metadata/aspect_catalog/catalog) to fresh files via Turso's stable VACUUM INTO. Lands in <base>/<label> where <base> is WEFT_BACKUP_DIR or <store_root>/backups and <label> is a traversal-guarded ?label= ([A-Za-z0-9][A-Za-z0-9._-]{0,99}: 1 to 100 letters, digits, ., _ or -, starting with a letter or digit and not ending with .) or, without one, a generated manual-<unix_millis>. The backup-<digits> form is reserved for the backup daemon (retention counts and prunes exactly those names), so a ?label= in that form is a 400, and no snapshot taken through this endpoint is ever pruned by retention. ?verify=source|snapshot picks the verification: source (the default) cross-checks each copy's user-table set + row counts against the live control plane — the strongest check, but it assumes a quiescent store; snapshot verifies each copy on its own terms (it reopens and fully scans every row of every table) without re-reading the source, which is what an online backup taken while ingest continues needs. An unknown token → 400. Returns per-DB {name, tables, rows, bytes} + total_rows/total_bytes + the verify mode that ran. The backup is built in a .partial-* directory beside its label and appears under the label, with a MANIFEST.json, only once complete and durable. A label outside that grammar → 400 (the grammar also keeps out .partial-*, .deleting-* and .restore-drill-*, the names of unfinished backups, prunes and drills); an existing target dir → 409 with "code": "already_exists"; no store → 503. Control plane only — the .weftseg measurement frames are not part of this backup (hard-constraint #3).
POST /api/v1/storage/restore/drill?label=<backup>Restore drill (Phase 7.4) — rehearse restoring a backup on a running server, so an operator can answer "is my backup actually restorable?" without risking anything. Restores the named backup into a throwaway directory beside the backups, verifies every restored database at its destination (it opens, and every row of every table reads), returns per-DB {name, tables, rows, bytes} + total_rows/total_bytes/restorable, then deletes the rehearsal copy (cleanup_error says why, if it could not; it is null otherwise). Non-destructive by construction — it cannot touch the live store, and restoring over a live control plane is deliberately not offered (the library primitive refuses to clobber; pointing WEFT_SEGMENT_STORE_ROOT at a restored copy is a deployment decision, not an HTTP call). Unknown backup → 404; a label outside the backup label grammar (so also traversal and the staging names .partial-*, .deleting-*, .restore-drill-*) → 400, while a daemon snapshot's backup-<digits> is accepted, since a drill only reads it; a rehearsal directory already taken by a drill started in the same millisecond → 409 with "code": "already_exists" (retry; nothing was restored or removed); a backup that will not restore → 500 carrying the failure, which is a failed drill rather than a server bug.
GET /api/v1/storage/{aspect}/stats · …/storage/statsMaterialized per-aspect and store-wide rollups, including the realized bytes/point (the north-star cost term), an unsorted_segments order-health count (segments that would force a linear scan on a point lookup), and an overlapping_segments count (time-overlapping segments — the cross-segment order-health signal, computed by an index scan). Rollup fields are served from the control plane without opening a segment.
\n

Metrics

\n

GET /metrics renders Prometheus text exposition (v0.0.4): shared\nweft_interpolate_* / weft_downsample_* counters across each endpoint family,\nweft_ingest_* counters (requests/errors/rows/segments) on the ingest paths,\nweft_reconcile_* counters (_passes_total / _segments_reconciled_total) over the\nout-of-order reconciliation passes (manual, store-wide, and the background daemon —\nthreshold-held calls excluded) plus _failed_passes_total, the per-aspect passes a\nstore-wide sweep (daemon tick or POST /api/v1/storage/reconcile) skipped after an\nerror (the daemon also logs one WARN line per failed aspect), weft_backup_*\ncounters (_snapshots_total / _bytes_written_total) over the online control-plane\nbackups, and latency histograms over the compute and seal paths.\nGET /debug/profile/current serves the same timing data as a live p50/p95/p99\nsnapshot.

\n

Backup & restore

\n

The control plane (the catalog, per-aspect schema, segment index and metadata\nrollup databases) has an online backup, two verification modes, an unattended\ndaemon, and a restore. Per the storage boundary this covers the control plane\nonly — the .weftseg measurement frames are not part of a snapshot.

\n
    \n
  • Online snapshot. POST /api/v1/storage/backup copies all four databases\nwith Turso's VACUUM INTO while the store stays fully usable, and verifies\nevery copy before returning (see the endpoint catalog above for the parameters).
    • \n
  • Two verifications, because "verified" means different things under load.\n?verify=source (default) cross-checks each copy against the live control plane\n— the strongest check, valid while the store is quiescent. ?verify=snapshot\nnever re-reads the source; it reopens each copy and fully scans every row of\nevery table, so it is correct while writers are committing. A snapshot-only\nbackup taken with six concurrent ingests in flight returns 200 with the\nconcurrent seals visible in the copy.
    • \n
  • A backup directory holds exactly its four databases and a MANIFEST.json.\nVerifying a snapshot reopens it, which leaves zero-byte -wal / -log journal\nsidecars beside the copy; these are swept after every verification, in both\nmodes. Only a sidecar that is exactly zero bytes is removed — a non-empty -wal\nholds un-checkpointed frames, which is data, and is never touched. Verification\nalso requires each database's own tables, so an empty file never passes.
    • \n
  • A backup appears only once it is complete and durable. It is built in a\n.partial-{label}-{nonce}/ directory; each database is verified and fsynced, the\nmanifest (format, creation time, each file's size, tables and rows) is written and\nfsynced, and the directory is fsynced, renamed to its label, and the parent\nfsynced. A backup that fails with an error before the rename removes its build\ndirectory again; a crash there leaves the .partial-* directory, which is never\ncounted or restored, and which the backup daemon's sweep removes (where the\ndaemon is not enabled, remove it by hand). Once renamed, only the parent's fsync\nis left: if that fails, the call reports an error (500 over HTTP) although the\ncomplete backup is already under its label, so a retry with the same label is\nrefused (409). The next backup's fsync of the same parent makes it durable.
    • \n
  • What a snapshot costs is dominated by the filesystem, not by the vacuum.\nBenchmarked in weftdb/benches/backup_cost.rs:\non tmpfs a whole four-database backup-and-verify runs in 8.62 ms at one\naspect, 6.18 ms at 16 and 16.79 ms at 128 (snapshot verification;\nsource verification is within noise of those), so the vacuum's own work is\nmilliseconds and scales with control-plane rows rather than being a fixed\nper-database floor. On a real disk it is orders of magnitude slower, because\nTurso's VACUUM INTO fsyncs the destination and runs a TRUNCATE checkpoint\nbefore returning. On this project's btrfs volume (load average ~12) the\nbackground daemon's ticks landed 3.0–4.5 s apart at a 2 s configured interval.\nA tick fires late only when the one before it outlasts the interval (tokio's\ndefault Burst catch-up), so each snapshot and its prune took that long, and\nthey ran back to back. Set WEFT_BACKUP_INTERVAL_SECS from what your storage\ncan sustain, not from how much data you have; point WEFT_BENCH_BACKUP_DIR\nat your own volume to measure it.
    • \n
  • Unattended backups with retention. Set WEFT_BACKUP_INTERVAL_SECS to run a\nbackground daemon that snapshots into a fresh backup-<unix_millis> directory\neach tick (verifying snapshot-only, since it backs up a live store), and\nWEFT_BACKUP_KEEP to retain only the newest N. Retention is deliberately narrow:\nit only ever removes daemon-generated backup-<digits> directories, a form\nthe API refuses to create (a directory you rename into it by hand is treated as\ngenerated and can be pruned, as it always could), so a snapshot you took\nthrough the API (?label=nightly, or an unlabelled manual-<unix_millis>) is\nnever a prune candidate and never counts toward N, and\nit runs only after a successful snapshot, so a run of failures cannot prune\nyour last good backup away. It counts only complete backups (a manifest, or all\nfour databases from before manifests), and renames a pruned backup to\n.deleting-* before removing its files, so an interrupted prune never leaves a\nhalf-removed backup. A backup-<digits> directory stamped more than 24 h\npast the clock, or past the directory's own modification time, did not come from\nthe daemon: retention neither counts nor removes it, and logs a warning each time\nit sees it. The modification-time check keeps such a directory ignored after the\nclock catches up with its stamp, as long as the directory is left unmodified and the\nfilesystem reports modification times; remove it by hand when the warning appears.\nAt start and on every tick the daemon removes .partial-*, .deleting-* and\n.restore-drill-* entries untouched for an hour. Both paths record\nweft_backup_* metrics.
    • \n
  • Restore, with a drill. weftdb::restore_control_plane(backup_dir, root)\nputs a snapshot back into a store root and verifies each restored file at its\ndestination — what matters is that the file the store will open actually\nreads. It refuses an incomplete backup directory (or one whose files no longer\nmatch its manifest, or one with a staging name) and refuses to overwrite an\nexisting control plane, pre-flighting both across all four files before writing\nanything, so a rejected restore leaves nothing behind. Each file is copied to\n<name>.tmp, fsynced and verified; only once all four verify are they renamed\ninto place, one after another, and the root fsynced. A failure before the\nrenames leaves only .tmp files, which a retry replaces; one part-way through\nthem leaves some databases in place, which a retry refuses to overwrite, so clear\nthe root first. Restoring beside the\noriginal segments/ frames reconstitutes a working store: the round-trip test\nreopens the restored root and reads its measurements back.
    • \n
  • Drills on a live server. POST /api/v1/storage/restore/drill?label= rehearses\na restore of the named backup into a throwaway directory, verifies it, reports\nrestorable, and cleans up — non-destructive by construction, so the question\n"is my backup actually restorable?" can be asked of production. Restoring over\na live control plane is not offered: that is a deployment decision (point\nWEFT_SEGMENT_STORE_ROOT at a restored copy and restart), not an HTTP call.
    • \n
\n
\n

Configuration

\n

WeftDB is configured primarily through environment variables:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
VariableUsed byPurposeDefault
WEFT_DATA_DIRweftdb, weft-tuiRoot directory for database files and the TUI log (weft-tui.log).portable per-user default (see below)
TEST_DATA_DIRweftdbHighest-priority override for the database root (used by the test suite).unset
WEFT_EXTRACTED_BATCH_RETENTION_SECSweftdbHow long (seconds) a legacy aspect remembers the batches pattern extraction consumed, so the incremental build does not queue them again. Each extraction deletes older records, keeping about one row per resolution step of this period. Set it above the longest interval between pipeline runs of an aspect plus the longest ingest call; a non-positive or unparsable value keeps the default.172800 (48 hours)
WEFT_SERVER_ADDRweft-serverHTTP bind address.127.0.0.1:8080
WEFT_ALLOW_ANY_HOSTweft-serverTruthy (1/true/yes/on) → turn off the loopback request guard. On a loopback bind the server otherwise answers only requests whose Host is localhost, 127.0.0.0/8 or [::1] (421 otherwise) and refuses state-changing requests from non-loopback web origins (403). Set it behind a local reverse proxy that forwards a different Host, or one that forwards the non-loopback Origin of a browser UI it fronts. A non-loopback bind is never guarded.unset (guarded on a loopback bind)
WEFT_GPU_CALIBRATEweft-serverStartup calibration. By default, once the listener is bound, the server runs splimes::calibrate() once in the background on a blocking thread: it starts the GPU if there is one, times the single-thread, rayon and GPU backends on grids up to 16 Mi points (several seconds, a few hundred MB), and sets where Backend::Auto switches between them. Requests are served from the start and interpolate on the CPU with splimes' default thresholds until it finishes. A CPU/software adapter (llvmpipe, lavapipe, WARP) is not calibrated, with a log line saying why; force calibrates it anyway. 0 (or false/no/off) skips calibration. It logs the adapter (or why there is none) and the thresholds, and never fails startup; skipped or failed, interpolation stays on the CPU with splimes' defaults.unset (calibrate, except a software adapter)
WEFT_MAX_INTERPOLATE_POINTSweft-serverThe most output points one /api/v1/interpolate* request may produce; a larger grid is a 400 naming its size and the limit, refused before anything is allocated. A positive integer, read once at startup; anything else (including 0) stops the server from starting with a message naming the variable.10000000
WEFT_SEGMENT_STORE_ROOTweft-serverRoot of the Storage v2 segment store; enables the /storage endpoints.unset (storage endpoints answer 503)
WEFT_ON_AMBIGUOUS_COMMITweft-serverWhat the server does once its segment store is write-poisoned (after a COMMIT that may or may not have committed): poison keeps serving, refusing every write until the server restarts while reads keep working and GET /ready reports poisoned; exit logs and exits with status 70, for a supervisor that restarts the server. The restart's open settles the transaction either way. Read once, at startup; the weftdb library never reads it and never exits the process.unset (poison)
WEFT_RECONCILE_INTERVAL_SECSweft-serverBackground reconcile daemon sweep interval in seconds; 0/unset disables it.unset (disabled)
WEFT_RECONCILE_THRESHOLDweft-serverunsorted_segments backlog an aspect must reach before the daemon reconciles it.1
WEFT_RECONCILE_HOT_COLDweft-serverTruthy → the daemon reconciles cold segments each tick and defers the hot tail until the threshold.unset (all-or-nothing)
WEFT_RECONCILE_OVERLAPSweft-serverTruthy → each daemon tick also merges cross-segment time-overlap groups.unset (disabled)
WEFT_RECONCILE_SPLIT_MIN_BYTESweft-serverSplit-not-rewrite floor (bytes) for the daemon's overlap merge; a dominant cold prefix clearing it is split off rather than rewritten.unset (default 50 MiB floor → full-rewrite)
WEFT_RECONCILE_MAX_SPLITSweft-serverSegment-count cap; each tick also squashes every aspect over it into one segment (bounds split-path fragmentation).unset (no squash)
WEFT_COMPACT_TARGET_ROWSweft-serverTarget segment size in rows for the daemon's size-aware compaction pass; each tick coalesces every aspect's segments toward ~this many rows per segment (leaving already-large segments alone), holding fragmentation near the read-optimal size instead of folding to one (WEFT_RECONCILE_MAX_SPLITS). Motivated by the downsample_range knee (one giant segment reads slower than several mid-sized ones). Needs the daemon interval set.unset (no compaction)
WEFT_BACKUP_DIRweft-serverDirectory each control-plane backup is written under — shared by the manual POST …/storage/backup endpoint and the background backup daemon, so hand-taken and automatic snapshots live together.<store_root>/backups
WEFT_BACKUP_INTERVAL_SECSweft-serverBackground control-plane backup daemon interval in seconds; 0/unset disables it. Each tick writes a verified snapshot into a fresh backup-<unix_millis> directory, using the concurrent-write-safe snapshot verification (it backs up a live store). Needs a segment store configured.unset (disabled)
WEFT_BACKUP_KEEPweft-serverHow many daemon-generated snapshots to retain; after each successful backup the oldest backup-<digits> directories beyond this many are removed. Only daemon-generated names are ever counted or pruned — a snapshot taken through POST …/storage/backup (a ?label= one, or an unlabelled manual-<unix_millis> one) is never a candidate — and a generated-looking directory stamped more than 24 h past the clock or past its own modification time is ignored (not counted, not removed, logged as a warning; remove it by hand).unset (retain everything)
BTC_TEST_MAX_ROWS / BTC_TEST_FULLweftdb (tests)Row cap for the bounded, hermetic default run of test_create_btc_1min_database (into a temp data dir), or BTC_TEST_FULL=1 for the historical whole-corpus load into the shared data dir.5000 / unset
WEFT_SEGMENT_CHECKPOINT_STRIDEweft-serverRows between entries of the sealed timestamp checkpoint index — trades a little size for much faster point lookups on sorted, irregular columns (~3.45× single-block; see the checkpointed-frames feature above). Applies only where it pays: sorted + irregular + at least WEFT_SEGMENT_CHECKPOINT_MIN_ROWS rows. ~1024 is the sweet spot (stride barely moves speed but does move size).unset (no index; frames byte-for-byte as before)
WEFT_SEGMENT_CHECKPOINT_MIN_ROWSweft-serverRow floor below which a segment is never checkpointed (a small column decodes trivially, so an index would be pure cost).8192
WEFT_SEGMENT_CHECKPOINT_MAX_CODEC_OVERHEADweft-serverCeiling on the timestamp-codec override a checkpointed seal will accept (blocked / best bytes; 1.0 = only when free). A checkpointed frame must use the range-decodable per-block codec, which is ~free where that codec already wins but ~3.5× on a Gorilla-shaped and ~14× on an RLE-shaped column — this refuses those seals rather than silently bloating them.1.25
WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEADweft-serverCeiling on the size overhead the bit-sliced (transposed) value codec may pay against the size-selected codec (transposed / best bytes; 1.0 = only when free, and a value below 1.0 is meaningful — the bit-sliced layout can be a strict size win, since it pays one width header per 1024-value tile where the blocked codec pays one per 64 values). Stores the value column bit-plane-major. Measured byte-neutral end to end on a 1M-row column and no faster to read than the linear codec (see Where it doesn't pay off), so it is off by default. Needs a build with the bitsliced-codec feature; any other build ignores the variable and logs a warning.unset (no transposed codec; frames byte-for-byte as before)
WEFT_SEGMENT_PARTIAL_BASEweft-serverResolution token (seconds/minutes/hours/…) at which each sealed segment materializes a partial-reduction .weftpart sidecar — a stored mergeable partial of the bounded reductions. A stored-range downsample of those reductions then merges the sidecars instead of decoding the value column (measured 3.2×), re-keyed to any coarser nesting resolution. A sealed segment is immutable, so the sidecar never goes stale; a rewrite (reconcile/split/squash) regenerates it.unset (no sidecar; downsample_range decodes as before)
WEFT_SEGMENT_PARTIAL_MIN_ROWSweft-serverRow floor below which a segment gets no partial sidecar (a tiny segment's partial saves too little decode to be worth the extra file).4096
WEFT_SEGMENT_PARTIAL_TIERSweft-serverComma-separated fine→coarse rollup resolutions (e.g. hours,days) materialized beside the WEFT_SEGMENT_PARTIAL_BASE partial, each re-keyed from the tier below (up to four kept; a finer/non-nesting entry is skipped). A coarse stored-range downsample then folds the coarsest matching tier instead of re-keying the whole fine base (measured ~1.16× on a DAY query over a MINUTES base). No effect unless WEFT_SEGMENT_PARTIAL_BASE is set.unset (base-only sidecar)
RUST_LOGalltracing filter. On weft-server it drives a per-request root span (request{method,path,request_id}, echoed as x-request-id) that every per-stage span nests under, each with busy/idle timing: the compute paths (interpolate.parse/compute/serialize under interpolate.engine, downsample.parse/reduce); the storage read paths (storage.{range,value_range,point}.read + .serialize, with a format field over JSON/CSV/Arrow/Parquet); the ingest paths (storage.ingest.parse/normalize/seal for ILP/CSV/JSON, and storage.ingest.parquet for the Parquet decode+seal); the background reconcile daemon (reconcile.tick{kind,…,aspects,segments}); and the control-plane writes on the seal path (control_plane.index.insert{aspect,id,rows_changed}, control_plane.index.delete{…}, control_plane.metadata.put{aspect,rows_changed}), whose rows_changed is libSQL's own affected-row count (Statement::n_change()) rather than an inference — so a trace shows whether a catalog write actually changed anything.weft_tui=debug,database=debug,info
OTEL_EXPORTER_OTLP_ENDPOINTweft-serverWhen set (e.g. http://localhost:4317), export tracing spans to an OpenTelemetry collector over OTLP/gRPC in addition to the RUST_LOG fmt output. Unset → no exporter, no network dependency; a misconfigured/absent collector never blocks startup. scripts/verify-otlp.sh verifies delivery end-to-end against a local Jaeger container (starting one if needed).unset (export disabled)
SKIP_SLOW_TESTStestsSet to 1 to skip long-running tests.unset
\n

The WEFT_SEGMENT_* variables configure the store weft-server opens; the server maps\nthem to weftdb::SegmentStoreOptions. A program embedding weftdb gets them only by\nasking: SegmentStore::open uses SegmentStoreOptions::default() (every one of them\nunset), and SegmentStore::open_with_options(root, SegmentStoreOptions::from_env())\nreads them as the server does.

\n

The database root directory is resolved in this order:

\n
    \n
  1. TEST_DATA_DIR, if set (explicit override for the test suite);
    2. \n
  2. otherwise WEFT_DATA_DIR, if set;
    3. \n
  3. otherwise a portable, per-user default — ~/.weftdb/data (falling back to the\nplatform data directory + weftdb, e.g. %APPDATA%\\weftdb on Windows, if the home\ndirectory cannot be determined).
    4. \n
\n

No paths are hard-coded: data lands in a writable, machine-independent location out\nof the box, and setting WEFT_DATA_DIR relocates both the databases and the TUI log\ntogether. The resolution functions are exported as weftdb::data_dir() (resolved)\nand weftdb::default_data_dir() (the raw default).

\n

The splimes crate also exposes build features:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
FeatureEffect
gpu (default)The wgpu backend. Without it, interpolation always runs on the CPU.
serde (default)Serialize/Deserialize for Point, PointKind, Resolution and Spline.
tokioInterpolator::run_async, which runs an interpolation on tokio's blocking pool. WeftDB enables it: every async caller (the weftdb read and compression paths, weft-server, the bench adapter) interpolates off the async workers.
\n

The GPU is started at runtime, not at build time: weft-server calibrates it in the\nbackground at startup (WEFT_GPU_CALIBRATE, above), and an embedding program calls\nsplimes::calibrate() or splimes::prewarm_gpu() itself.

\n

WeftDB's own build features are all off by default, sit outside the 1.0 semver\npromise, and are pending patent review. A default build reads every segment written\nunder the default configuration (WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEAD unset). A segment\nwritten with that variable set, as builds from before v0.1.0 did whenever it was set, needs\nbitsliced-codec. The release binaries are built without either feature, except that\nweft-bench enables experimental-codecs.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
FeatureCrate(s)Effect
bitsliced-codecweft-physical-type, forwarded by weftdb and weft-serverCompiles in the opt-in bit-sliced value codec, so WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEAD takes effect. Without it, a segment written with the codec fails to read with an error naming the feature.
experimental-codecsweft-physical-type (enabled by weft-bench)Advisory codecs never written to disk: Gorilla-XOR, Chimp, Chimp128 and Elf for f64, and the Sprintz FIRE timestamp forecaster. Benchmark what-ifs only.
\n
\n

Rust library

\n

WeftDB can be used directly as a set of libraries. They are not on crates.io yet\n(v0.1.0 ships binaries only), so depend on them from this repository, pinned to a\nrelease tag. splimes, the interpolation engine, is on crates.io:

\n
[dependencies]\n# the database: capture, query, interpolate\nweftdb = { git = \"https://github.com/basic-automation/weftdb\", tag = \"v0.1.0\" }\n# the analytics pipeline\nweft-orchestration = { git = \"https://github.com/basic-automation/weftdb\", tag = \"v0.1.0\" }\n# the interpolation engine on its own\nsplimes = \"1\"
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
CrateDepend on it when you want
weftdbThe database — capture measurements, query and interpolate stored series.
splimesSpline interpolation over irregular series, standalone. No database.
weft-orchestrationThe batching → patterns → events → correlation → signals pipeline.
weft-physical-typeDeclared numeric encodings and the .weftseg columnar format.
weft-reduceDownsampling reductions over an epoch-aligned bucket grid.
weft-line-protocolTo parse InfluxDB Line Protocol without an InfluxDB client.
weft-arrowSegments as Apache Arrow RecordBatch, Arrow IPC or Parquet.
weft-arrow-storeA stored range read straight into an Arrow RecordBatch.
\n

weft-server, weft-tui and weft-bench are binaries rather than libraries, so they\nare not published to crates.io — take them from the archives on a\nGitHub release or build them\nfrom this repository.

\n

The examples below are illustrative — run cargo doc --open for the authoritative,\nversion-matched API.

\n

Capture Measurements

\n
use weftdb::{Database, DatasetId, InputMeasurement, Resolution};\nuse bigdecimal::BigDecimal;\nuse std::str::FromStr;\nuse chrono::{TimeZone, Utc};\n\n#[tokio::main]\nasync fn main() -> anyhow::Result<()> {\n    let db = Database::new(\"my_sensors\").await?;\n    let sensor = db.observe_subject(\"temperature_sensor_001\").await?;\n    let temp = db.track_aspect(&sensor.id(), \"ambient_temp\", &Resolution::Seconds, None).await?;\n\n    let measurements = vec![\n        InputMeasurement::new(Utc.with_ymd_and_hms(2024, 1, 1, 12, 0, 0).unwrap(), BigDecimal::from_str(\"22.5\")?),\n        InputMeasurement::new(Utc.with_ymd_and_hms(2024, 1, 1, 12, 1, 0).unwrap(), BigDecimal::from_str(\"22.8\")?),\n    ];\n    db.batch_capture_measurements(temp.id(), DatasetId::new(), measurements).await?;\n    Ok(())\n}

Query & Interpolate

\n
use weftdb::{Database, Resolution, Spline};\nuse chrono::{TimeZone, Utc};\nuse futures::StreamExt;\n\nlet db = Database::existing(\"my_sensors\").await?;\n\n// Interpolated point at a specific time:\nlet point = db.analyze_point(&aspect_id, query_time, &Resolution::Seconds, &Spline::Linear).await?;\n\n// Streamed, interpolated range:\nlet mut stream = db.analyze_range(&aspect_id, start, end, Resolution::Seconds, Spline::Linear).await?;\nwhile let Some(result) = stream.next().await {\n    let p = result?;\n    println!(\"{}: {}\", p.timestamp, p.value);\n}

Run an Analytics Pipeline

\n
use weft_orchestration::Pipeline;\nuse weftdb::Database;\nuse splimes::Spline;\n\nlet database = Database::existing(\"PlantTelemetry\").await?;\n// ... resolve `aspect_id` for the \"outlet_pressure\" aspect of pump-station-3 ...\n\nlet mut pipeline = Pipeline::builder(database.clone(), aspect_id)\n    .spline_method(Spline::Linear)\n    .batch_size(24)                              // 24-hour batches\n    .add_dictionary(\"daily\", \"Daily patterns\", constraints)\n    .with_monthly_increase_detector(0.05)        // 5% monthly increase\n    .with_peak_detector(\"Pressure Peaks\")\n    .build()\n    .await?;\n\npipeline.run().await?;                            // full pipeline (incremental)\nprintln!(\"Signals generated: {}\", pipeline.signals().len());

For finer control, the individual steps — prepare_data, extract_patterns,\ndetect_events, correlate_events, generate_signals — can be called directly.

\n
\n

Terminal UI

\n

weft-tui is the interactive way to explore WeftDB (the HTTP server is the primary\ncommercial surface; the TUI serves demos and debugging). Launch it with\ncargo run --release -p weft-tui. It is a stateful, keyboard-driven application that\nwalks you through:

\n
    \n
  • Database / Subject / Aspect creation — with name validation and resolution\nselection (+/- to step through resolutions from nanoseconds to years).
    • \n
  • CSV import — point it at a CSV file and watch live loading progress\n(rows loaded / scanned).
    • \n
  • Browsing — select an existing database, subject, and aspect.
    • \n
  • Plotting — visualize an aspect's series directly in the terminal.
    • \n
  • Compression — choose a mode (time-based, size-based, or combined), configure\ntiers/targets/aggressiveness in a form, and monitor a live compression dashboard\n(phase, tier progress, aggressiveness, compression ratio) plus last-run stats.
    • \n
\n

Logs are written to weft-tui.log (see WEFT_DATA_DIR), and the last 50 log lines\nare also shown in-app.

\n
\n

How it works

\n

The parts below are reference material — you do not need them to use WeftDB.

\n

Architecture

\n
┌──────────────────────────────┐  ┌──────────────────────────────────────┐\n│           weft-tui            │  │              weft-server               │\n│  Ratatui terminal UI (app)   │  │  axum HTTP API: interpolate ·         │\n│  create DB · import · plot   │  │  downsample · storage ingest/query ·  │\n│  · compress                  │  │  catalog · /metrics · /debug/profile  │\n└──────────────┬───────────────┘  └──────┬───────────────────┬───────────┘\n               │ uses                     │ uses              │ columnar I/O\n               ▼                          ▼                   ▼\n┌──────────────────────────────┐   ┌────────────────────────────────────┐\n│    weft-orchestration    │   │   weft-arrow / weft-arrow-store      │\n│  batch → patterns → events   │   │   Arrow IPC + Parquet interchange  │\n│  → correlations → signals    │   └──────────────────┬─────────────────┘\n└───────────────┬──────────────┘                      │\n                ▼                                     ▼\n┌─────────────────────────────────────────────────────────────────────┐\n│                              database                                │\n│   Turso/libSQL control plane (catalog · metadata · segment index)   │\n│   + typed columnar `.weftseg` segment store (measurement hot path)   │\n│   Subject/Aspect/Measurement · Patterns · Events · Compression      │\n└───────────────┬─────────────────────────────────────┬───────────────┘\n                │ physical types + segments           │ numerics\n                ▼                                     ▼\n┌──────────────────────────────┐   ┌──────────────────────────────────┐\n│      weft-physical-type       │   │             splimes              │\n│  encodings · schemas ·       │   │   Spline interpolation engine    │\n│  timestamp codecs · .weftseg  │   │  CPU · Rayon · GPU (calibrated)  │\n└──────────────────────────────┘   └──────────────────────────────────┘

weft-bench drives the whole stack from outside through the same public APIs a\ncustomer would use; weft-line-protocol is the shared ILP dialect between the\nharness and the server. Dependencies flow downward; the arrow-* tree lives only\nin the weft-arrow* leaf crates.

\n
\n

Data Model

\n

WeftDB organizes data hierarchically. Each level maps to a directory or database file\non disk, so an aspect's storage is fully self-contained.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ConceptDescriptionExample
DatabaseTop-level container for all data.Crypto, my_sensors
SubjectA logical entity being observed.pump-station-3, temperature_sensor_001
AspectA specific measurement type of a subject, with its own resolution, declared physical encoding, and (optional) compression config.open, ambient_temp
MeasurementA timestamped, arbitrary-precision value.(2024-01-01T12:00Z, 22.5)
\n
On-Disk Layout
\n
{data_dir}/\n└── {database_name}/\n    ├── metadata.db                  # database metadata\n    └── {subject_name}/\n        └── {aspect_name}/\n            ├── measurements.db        # raw measurements\n            ├── unprocessed_batches.db # batching queue\n            ├── processed_batches.db\n            ├── patterns.db            # extracted patterns\n            ├── events.db              # detected events\n            ├── correlations.db        # pattern↔event links\n            ├── pipeline.db            # persisted pipeline config & state\n            └── dictionaries/\n                └── {dictionary_name}.db

The Storage v2 segment store (see\nPhysical Types & Columnar Storage) adds, per\nstore root: catalog.db (database/subject/aspect hierarchy + declared schemas),\nper-aspect metadata.db rollups, segment_index.db (the prune-before-open\nsegment index), and segments/*.weftseg (the typed columnar measurement files).

\n

Higher-level analytics types build on this foundation:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
TypePurpose
BatchA group of measurements processed together.
PatternAn extracted recurring shape, with the occurrences where it appears.
DictionaryA collection of patterns sharing similarity constraints.
EventA detected occurrence with one or more manifestations (time spans).
CorrelationA link between a pattern and an event.
SignalA probability-weighted prediction derived from a correlation.
\n
\n

Interpolation Engine

\n

splimes is the numerical heart of WeftDB. Given a set of Points and a target time\nrange + resolution, it produces an interpolated series using one of four spline\nmethods, choosing the execution backend automatically, and labels every output point\nRaw, Interpolated or Extrapolated (PointKind). It interpolates and\nextrapolates — single-instant lookups and full ranges — on the CPU (one thread or\nrayon's pool) and the GPU. WeftDB depends on splimes 1.0; its own\nmigration guide\nlists what changed from 0.1.

\n
Spline Methods
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
MethodMin. pointsNotes
Linear2Straight-line interpolation.
Quadratic3Second-degree.
Cubic4Smooth third-degree splines.
Polynomial(degree, bounds_factor)degree + 1Degree 1–8, with optional bounds damping of the extrapolation.
\n

If a method needs more distinct points than are available, splimes steps down\nautomatically (Cubic → Quadratic → Linear; Polynomial(d) → Polynomial(n − 1)),\nnever upgrading beyond what you requested, and reports the method it used.

\n
Backend Selection
\n

Every call is synchronous; Backend::Auto (the default) picks the backend by output\ngrid size, using AutoThresholds:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Grid sizeBackend
below parallel_min_points (default 64 Ki)CPU, the calling thread
from parallel_min_pointsParallel, rayon's pool
from gpu_min_points (default never), once the GPU has been startedGPU (wgpu compute shaders), falling back to rayon if it fails
\n

Auto never starts the GPU itself. splimes::calibrate() starts it, times every\nbackend on this machine and sets the thresholds to the measured crossovers;\nweft-server runs it once in the background at startup, skipping a CPU/software\nadapter (WEFT_GPU_CALIBRATE=0 skips it, force calibrates any adapter). Otherwise:

\n
    \n
  • splimes::prewarm_gpu() starts the GPU and returns its GpuInfo (adapter, API,\ndevice type, driver, f64 support); then set thresholds with\nsplimes::set_auto_thresholds.
    • \n
  • splimes::configure_gpu(GpuConfig) — before the GPU's first use — sizes its\nbuffer pool and per-dispatch chunk (max_pool_bytes, chunk_points, low_power;\npresets GpuConfig::minimal(), low_memory(), DEFAULT, high_performance()).\nCalling it after the GPU has started is an Error::GpuAlreadyConfigured, never a\nsilent no-op; splimes::gpu_config() reports what is in force.
    • \n
  • splimes::gpu_pool_stats() returns the buffer pool's counters once the GPU is up.
    • \n
  • The GPU computes in f64 where the adapter supports it; f32 must be asked for\n(Interpolator::gpu_precision(Precision::F32)).
    • \n
\n
use splimes::{Interpolator, Resolution, Spline};\n\n// Backend::Auto picks the backend by grid size; from async code, run it off the\n// executor (`run_async` needs the `tokio` feature):\nlet series = Interpolator::new(Spline::Cubic, Resolution::Seconds).run_async(points, start, end).await?;\nfor (timestamp, value, kind) in series.iter() {\n    // kind: PointKind::Raw | Interpolated | Extrapolated\n}
\n

Physical Types & Columnar Storage

\n

WeftDB's storage hot path is its own typed columnar format; Turso/libSQL serves as\nthe control plane (catalog, metadata, segment index, pipeline state) and never\nholds bulk measurements. BigDecimal remains the logical/API type everywhere.

\n
Physical type system (weft-physical-type)
\n
    \n
  • Six numeric encodings — F64, F32, ScaledI64, ScaledI128,\nDecimal128, BigDecimalText — each declaring storage width, hot-path/GPU\neligibility, and conversion behavior. Every encode reports an explicit\nExactness (exact / lossy-with-measured-residual); to_logical is always\navailable.
    • \n
  • No silent downcast, enforced — AspectSchema declares, per aspect, the\nphysical encoding, a permitted per-value error bound, and the timestamp unit.\nSealing a segment under a declared schema errors (SealError::Encode /\nSealError::ToleranceExceeded) rather than losing precision beyond the\ndeclared bound. Over HTTP this surfaces as a 400 — never a downcast.
    • \n
  • Advisory encoding selection — recommend_encoding picks the narrowest safe\nencoding within a tolerance, and columnar estimators report expected\nbytes/point (the north-star cost term) before anything is written.
    • \n
  • Timestamp codecs — integer-epoch timestamps (TimeUnit =\nseconds/millis/micros/nanos) with lossless delta and delta-of-delta\ntransforms and five interchangeable second-difference codecs — per-value\nzig-zag + LEB128 varint, RLE, fixed-width bit-packing, a\nGorilla-style variable-length codec, and per-block adaptive (dynamic)\nbit-packing — chosen per column by a smallest-wins selector. Each wins a different\nregime: bit-packing on regular/small-jitter series, RLE on long constant runs, Gorilla\non scattered single jitter (isolated moderate spikes among regular intervals, where\nRLE cannot form runs and bit-packing must widen every value), and per-block adaptive\nbit-packing on a mixed-magnitude stream (a contiguous wide region among narrow runs,\nwhere a single global width overpays — 184 B vs 384–961 B for the other codecs on a\n256-value mixed corpus). A regular 1000-point column packs to ~12 bytes total.
    • \n
  • Value-column codecs — a value column is stored under its schema-declared physical\nencoding, and a ScaledI64 column additionally chooses, via a self-describing\nselector byte, between a per-value zig-zag varint, fixed-width bit-packing of its\nmantissas, per-block adaptive (blocked) bit-packing, and per-block\nFrame-of-Reference (FOR) packing — bit-packing a regular scaled series (e.g. a\n2-decimal price ramp) to 37% below the varint, the blocked codec confining a wide\nburst to the blocks it spans on a mixed-magnitude column, and FOR (subtract each\nblock's minimum, pack the unsigned residual to the block's range — the standard\nFastLanes/ALP move) collapsing values clustered at a high base (a sensor reading\nnear a fixed offset: mantissas near 1e9 store 88% below the bit-packed\nalternative). Each codec is chosen only when strictly smallest, so a regular column is\nbyte-for-byte unchanged; the realized figure is reported as\nStorageEstimate.realized_value_bytes / value_codec. The timestamp column keeps an\nadvisory FOR estimate (for_estimated_bytes) — second differences are near-zero, so\nFOR rarely wins there. Every linear bit-packed codec decodes with a word-wise field read\n(one unaligned load and a shift/mask per value), so 1 Mi mantissas unpack in 1.09 ms\n(fixed width) / 1.83 ms (per-block)\n(weft-physical-type/benches/bitunpack.rs), and\n1 Mi real BTC closes stored under FOR at scale 8 decode to floats in 1.66 ms\n(weft-physical-type/benches/alp_vs_f64_codecs.rs).
    • \n
  • Two-level delta cascade (opt-in) — a trending ScaledI64 column (a counter or\nmonotone sensor whose magnitude every single-level codec pays for) is additionally\ncompressible by a cascade: delta-transform the mantissas, then pack the differences with\nthe smallest inner packer (varint / bit-pack / blocked / FOR / RLE). It is realized on disk\nas the VAL_CODEC_DELTA_CASCADE codec-chain descriptor (write_value_column_cascading,\nread back by the ordinary reader) and surfaced advisory-first in the bench\n(StorageEstimate.advisory_delta_cascade_value_bytes, schema v15). It is opt-in — the\ncascade beats even FOR broadly, so folding it into the default selector is a headline change\nheld for owner sign-off; the default codec choice is unchanged.
    • \n
  • Bit-sliced (transposed) value codec (opt-in, bitsliced-codec build feature) —\nVAL_CODEC_TRANSPOSED stores a\nScaledI64 column's mantissas bit-plane-major in 1024-value tiles, so the decoder rebuilds\neight lanes per plane byte with branch-free table spreads and a tile's width header drops the\nempty high bit-planes of a small-magnitude column. The decode kernel is checked against an\nexternal yardstick, the published fastlanes crate, at equal bit width\n(weft-physical-type/benches/fastlanes_yardstick.rs,\n1 Mi values): WeftDB's decoder takes 0.78 / 1.50 / 2.32 ms at widths 3 / 10 / 20 against\nfastlanes' 0.33 ms — still 2.4–7× behind, and that gap is an open roadmap item, not a\ndesign choice. It carries its own size function (transposed_value_bytes) and selector\nentry (best_value_codec_transposed(max_overhead)) rather than reusing the blocked figure, is\nrandom-access capable (so it does not regress the streaming point read), and is requested through\nFrameOptions / Segment::write_to_with or, from a deployment, WEFT_SEGMENT_TRANSPOSED_MAX_OVERHEAD.\nHonest result: no read-path win over the linear codec. Measured on a 1M-row zero-straddling\ncolumn (weft-physical-type/benches/transposed_read.rs,\n2026-10-08, transposed vs linear): frame bytes +0.08% (1,251,057 vs 1,250,076), full decode\n7.81 ms vs 6.87 ms; with the slicing-by-16 frame CRC, windowed 1000-row range 638 µs vs\n632 µs and single point read 820 µs vs 805 µs. So the layout is byte-neutral and no faster\n(slightly slower) on reads; it stays opt-in and the\ndefault codec choice is unchanged.\nNote the codec is chosen only when a strict size win or within the caller's overhead ceiling —\nthe ceiling may legitimately be set below 1.0, because per-tile widths with one header per 1024\nvalues can beat both a global width and the blocked codec's one-header-per-64. It is a\nbit-sliced layout, not the FastLanes layout (which keeps each value's bits together), and it\nis compiled only with the bitsliced-codec feature, pending patent review; without it the\nwriter never selects it and reading a segment that uses it fails with an error naming the\nfeature.
    • \n
  • Block-level random access — the fixed-layout value codecs support decoding a single value\n(or a sub-range) without materializing the whole column: weftseg::read_value_at(bytes, i)\nreads only the block covering row i (skipping earlier blocks by their headers) for the\nblocked/FOR codecs, reads bit i * width directly for the fixed-width bit-pack codec, and decodes\nonly the covering tile for the bit-sliced codec — the point-lookup / late-materialization lever.\nweftseg::read_value_range(bytes, start, len) is its windowed sibling: every fixed-layout codec\nlocates a value by walking its block/tile headers from the start of the stream, so resolving a\nwindow one value at a time re-walks that chain per row. The range read does the decode once,\nwhich is what keeps a windowed read on the 1024-value-tile bit-sliced layout from costing a whole tile\ndecode per row. weftseg::read_segment_point(bytes, t)\nwires this up to the framed single-block segment — it skips the value block by its framing,\ndecodes only the timestamps to find the row, and unpacks the one covering value block — so a\npoint lookup never materializes the value column on a per-block codec (equal to a full\ndecode + value_at for every frame; the non-block codecs fall back to that). read_paged_segment_point\ndoes the same for a paged frame, first pruning pages on their indexed min/max timestamp so only\nthe surviving page is touched. A regular (constant-stride) timestamp column is resolved in\nclosed form — ts[i] = first + i*step, so the row for an instant is O(1) with no timestamp\nmaterialization at all. Measured ~46× faster point lookup on a 100k-row single-block FOR\nsegment (56 µs vs 2.60 ms) and ~8.5× faster on the paged frame (33 µs vs 279 µs), with the\nclosed-form timestamp path a further ~3.4× over an irregular column (56 µs vs 188 µs), identical\nbytes on disk (weft-physical-type/benches/pointread.rs).
    • \n
  • Checkpointed frames — sublinear point lookup on an irregular sorted column (opt-in;\nwrite_segment_checkpointed / write_paged_segment_checkpointed). The closed form above only\nserves a regular column; an irregular one has no closed form, so a probe had to decode the whole\ndelta-of-delta timestamp stream. A checkpointed frame persists a sparse (row, timestamp, delta)\nindex every stride rows next to a range-decodable per-block dod stream, so a probe binary-searches\nthe index and decodes only the blocks it walks — the timestamp column is never materialized.\nMeasured ~3.45× faster point reads on a 100k-row sorted-irregular frame (352.94 ms → 102.42 ms\nfor 64 probes) for +0.41% frame bytes at stride 1024; a paged frame gains only ~1.14×\n(68.70 → 60.41 ms), because page pruning already bounds its decode to rows_per_page\n(weft-physical-type/benches/dodsearch.rs). The codec tag\nis additive — every previously written frame still reads, with no format-version bump — and a\ncheckpointed frame is byte-for-byte equivalent in behaviour to a plain one on every read.\nOpt in by setting WEFT_SEGMENT_CHECKPOINT_STRIDE (see the configuration table). The seal path\nthen writes the index only where it genuinely pays, on three counts: the column is sorted and\nirregular (a regular one already resolves O(1) closed-form, an out-of-order one cannot be\nbinary-searched), it has at least WEFT_SEGMENT_CHECKPOINT_MIN_ROWS rows, and the codec override\nis affordable — a checkpointed frame must use the range-decodable per-block codec, which costs\n~3.5× on a Gorilla-shaped column and ~14× on an RLE-shaped one, so those are refused rather than\nbloated (WEFT_SEGMENT_CHECKPOINT_MAX_CODEC_OVERHEAD). It is off by default, so an unconfigured\nstore's bytes/point are unchanged; making it the default is an open roadmap decision.
    • \n
  • Realized headline bytes/point — every headline bytes/point figure\n(Segment::bytes_per_point, StorageEstimate.bytes_per_point /\ntotal_bytes_per_point, the bench HTML val B/pt) reports the codec actually\nwritten to disk, not a fixed-width estimate (bench schema v10). The naive\nlen * width figure is retained beside it (Segment::logical_value_bytes,\nStorageEstimate.estimated_value_bytes) as the uncompressed baseline, so\nrealized / logical reads directly as the storage compression ratio.
    • \n
\n
Segment store (.weftseg + weftdb::SegmentStore)
\n
    \n
  • Sealed, checksummed segments — a Segment binds a typed value column and a\ndelta-of-delta timestamp column with min/max timestamp/value stats, row/null\ncounts, and a format version. The on-disk frame is hand-rolled, versioned, and\nCRC-32-verified before parse, so a corrupt or truncated file fails fast.
    • \n
  • Multi-codec timestamps — the timestamp block writes its second-difference\nstream under whichever of five codecs is smallest — fixed-width bit-packing,\nper-value zig-zag varint, RLE, Gorilla variable-length, or per-block\nadaptive (dynamic) bit-packing — chosen by a self-describing selector byte the reader\ndispatches on, routed through the single source of truth so the reported codec name\nalways matches the bytes on disk. A regular series packs to a handful of bytes on disk\n(a 1000-point regular block stores in <20 bytes), so the bytes/point saving is\nrealized, not just estimated. The value block carries its own codec selector too,\nso a ScaledI64 column stores its mantissas under whichever of varint, fixed-width\nbit-packing, per-block adaptive (blocked) bit-packing, or per-block\nFrame-of-Reference (FOR) packing is smallest on disk.
    • \n
  • Order semantics — every segment records whether its timestamps are monotonic\n(time_sorted). Ingest can enforce order (require_sorted rejects an\nout-of-order batch rather than sealing it), and the per-aspect/store rollups carry\nan unsorted_segments count so out-of-order data is visible before it costs a\npoint-lookup scan.
    • \n
  • Order-signal read planner — a single-instant point lookup\n(SegmentStore::read_point) prunes the index to the segments spanning the instant\nand resolves each with its persisted time_sorted flag: a sorted segment is\nbinary-searched, an out-of-order one linear-scanned (the only sound search on\nunsorted timestamps). Both frame kinds resolve through the streaming point read\n(weftseg::read_segment_point / read_paged_segment_point, below) — no value-column\nmaterialization on a per-block codec, and a paged frame prunes pages on their indexed\nmin/max timestamp without decoding a column byte before touching the one surviving page.\nSegmentStore::read_points resolves a batch of instants in one pass — the index is pruned\nonce and each segment's timestamp column decoded once for the whole batch, so N instants\nsharing a segment cost one decode, not N (~54× faster for 64 instants on a 100k-row FOR\nframe than 64 single lookups — 67 µs vs 3.57 ms,\nweft-physical-type/benches/pointread.rs).
    • \n
  • Intra-segment reconciliation — SegmentStore::reconcile_segment/reconcile_aspect\nrewrite an out-of-order segment into a sorted one in place (stable sort by\ntimestamp, re-sealed at the same id, frame kind preserved), so it drops out of the\nunsorted_segments count and its point lookups binary-search.
    • \n
  • Threshold-triggered reconciliation — the unsorted_segments backlog drives a\nQuestDB-style trigger so the rewrite is paid only once out-of-order data is worth\ncompacting, never on every late row: reconcile_aspect_if_unsorted_exceeds gates a\nsingle-aspect pass on the backlog, reconcile_all_over_threshold sweeps every\ndeclared aspect over the threshold, and a background timer daemon\n(WEFT_RECONCILE_INTERVAL_SECS / WEFT_RECONCILE_THRESHOLD) runs the sweep\nunattended. A threshold of 0 clamps to 1 (any out-of-order segment). Passes and the\nsegments they rewrite are counted in weft_reconcile_* metrics.
    • \n
  • Hot/cold reconciliation — reconcile_aspect_hot_cold/reconcile_all_hot_cold\nreconcile every cold (sealed) out-of-order segment on each pass but defer the\nhot tail (the most-recently-sealed segment) until the backlog reaches the\nthreshold, so the actively-appended segment is not rewritten on every tick —\nQuestDB's "squash non-active partitions each commit, defer the active one" policy.\nEnabled on the daemon with WEFT_RECONCILE_HOT_COLD and on the reconcile endpoints\nwith ?hot_cold=true.
    • \n
  • Cross-segment overlap signal — a sorted segment whose time window a\nlater-sealed segment re-enters (late data landing in an already-covered window) is\nnot counted by unsorted_segments; SegmentIndex::overlapping_count reports these\ntime-overlapping segments as a distinct overlapping_segments order-health\ncount in the per-aspect and store-wide rollups.
    • \n
  • Cross-segment overlap merge — SegmentStore::reconcile_overlaps/reconcile_all_overlaps\ncollapse each connected group of time-overlapping segments into one time-sorted\nsegment, folding members oldest→newest with newer-wins (last-writer-wins /\nupsert) dedup on shared timestamps — the same answer a point lookup already gives\nacross overlaps, and QuestDB's DEDUP UPSERT "last write wins on the designated\ntimestamp" semantics. Drives overlapping_segments to zero; exposed on the reconcile\nendpoints with ?overlaps=true and as a background daemon sweep\n(WEFT_RECONCILE_OVERLAPS).
    • \n
  • Split-not-rewrite merge — SegmentStore::split_segment carves a sorted segment\nat a boundary into a cold prefix (kept id) + hot suffix (new id), and\nreconcile_overlaps_with_policy consults a size-based SplitPolicy: when an overlap\ncomponent's cold prefix clears a byte floor and outweighs its hot suffix, only the\nsuffix is merged and the large cold prefix is left untouched, so later late arrivals\nre-entering the hot window never rewrite it — QuestDB's partition-split write-amp\nbound. The default reconcile_overlaps keeps QuestDB's 50 MiB floor (small components\nfull-rewrite); a floor is selectable with ?split_min_bytes= on the per-aspect and\nstore-wide reconcile endpoints and WEFT_RECONCILE_SPLIT_MIN_BYTES on the daemon.
    • \n
  • Squash — SegmentStore::squash_aspect/squash_aspect_if_exceeds/\nsquash_all_over_threshold fold an aspect's segments back into one (newer-wins),\nbounding the fragmentation repeated split carve-offs create — QuestDB's\nmax.splits squash trigger. Exposed as POST …/{aspect}/squash?max_segments= and on\nthe daemon with WEFT_RECONCILE_MAX_SPLITS (squash aspects over the cap each tick,\nafter the overlap merge that accumulates the splits).
    • \n
  • Null/quality column — a per-row presence bitmap (zero bytes for fully dense\ncolumns) makes gaps real: present values are stored densely and reconstructed\nas None on read.
    • \n
  • Paged segments — rows partition into fixed-height pages, each independently\nencoded with its own stats, behind a per-page index table: a reader prunes\npages on min/max time/value without touching a column byte and seeks\nstraight to the pages it needs.
    • \n
  • Data skipping — segment-level and page-level pruning by time, by value\nband, and by quality (segments/pages that overlap a window but hold only\nnulls there are skipped). A range read over a regular (constant-stride)\nblock-coded segment goes further — read_segment_range computes the row\nwindow in closed form and unpacks only the present values inside it, so a\nselective range decodes ~window values, not the whole segment (~47× faster\nfor a 100-row window over a 100k-row FOR frame — 58 µs vs 2.73 ms,\nweft-physical-type/benches/pointread.rs).
    • \n
  • Control plane — SegmentIndexStore persists one descriptor per sealed\nsegment in libSQL and answers range queries with SQL pruning;\nCatalogStore/AspectCatalog register the database → subject → aspect\nhierarchy with declared schemas (declare once, seal by lookup — an undeclared\naspect is refused, not guessed); per-aspect metadata.db rollups make\naspect-wide stats (including realized bytes/point) an O(1) read, always\nre-derivable from the segment index.
    • \n
\n
Arrow & Parquet interchange (weft-arrow, weft-arrow-store)
\n
    \n
  • Lossless Arrow batches — sealed segments and stored reads convert to/from\nApache Arrow RecordBatches: a lossless text form, plus a typed fast path\n(F64 → Float64, F32 → Float32, scaled integers → exact Decimal128).\nPaged segments stream one batch per page.
    • \n
  • Wire formats — Arrow IPC stream bytes and Parquet files (embedding\nthe Arrow schema so WeftDB's time-unit/encoding metadata survives), both\ndirections: export a stored range, or ingest a Parquet file into a declared\naspect under the full no-silent-downcast guarantee.
    • \n
  • Lean core preserved — the arrow-*/parquet dependency tree lives only in\nthese leaf crates; splimes/weftdb/weft-physical-type stay arrow-free.
    • \n
\n
\n

The Analytics Pipeline

\n

The weft-orchestration crate provides a fluent Pipeline API. Each aspect\nhas exactly one pipeline, persisted to its pipeline.db, so runs are incremental\nand conflict-free. A pipeline can host multiple dictionaries and multiple detectors.

\n
 Measurements ─▶ Batches ─▶ Processed Batches\n                                   │\n                                   ▼\n                          Dictionaries (1..N)        Event Detectors (1..N)\n                          extract patterns           peaks · valleys ·\n                                   │                  thresholds · monthly · …\n                                   ▼                          │\n                               Patterns ───── correlate ──────┤\n                                                              ▼\n                                  Events ─▶ Correlations ─▶ Signals\n                                                              │\n                                                              ▼\n                                          query_probability(event, time)

Built-in event detectors (in weft_orchestration::detectors):

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
DetectorDescription
detect_monthly_increaseMonths where the value rises ≥ a threshold start-to-end.
detect_peaks / detect_all_peaksGlobal / all local maxima.
detect_valleys / detect_all_valleysGlobal / all local minima.
detect_threshold_crossing_up / _downUpward / downward threshold crossings.
detect_drawdownSignificant value drops.
\n

You can also register custom detectors with the event_detector_fn! macro, and\nrun pipelines across many aspects in parallel with run_all_pipelines /\nrun_subject_pipelines.

\n
\n

Dataset Compression

\n

Long-running deployments accumulate huge volumes of raw measurements. The\ncompression module (weftdb::compression) reduces storage while preserving the\nshape of the data, configured per-aspect via CompressionConfig:

\n
    \n
  • Time-based — recent data within a "pure" window (e.g. the last 7 years) stays\nuncompressed; older data is progressively compressed across configurable tiers\nwith Linear, Exponential, or Custom aggressiveness scaling.
    • \n
  • Size-based — compress until total storage fits within a target (e.g. 10 GB),\nhitting the oldest data hardest. Takes precedence over time-based.
    • \n
\n

Compression works by interpolating to a coarser resolution and applying\nslope-change simplification to drop redundant points. Newly inserted data inside a\ncompressed range marks a dirty region for targeted re-compression. Progress is\nreported live (phase, tier, aggressiveness, time range) — surfaced in the TUI.

\n
use weftdb::{CompressionConfig, TimeBasedCompressionConfig};\n\nlet config = CompressionConfig::time_based(\n    TimeBasedCompressionConfig::default_seven_years()\n);\nlet aspect = db.track_aspect(&subject_id, \"temperature\", &Resolution::Hours, Some(config)).await?;\nlet summary = aspect.compress(&db).await?;
\n

Benchmarking

\n

Weft-Bench (weft-bench) is WeftDB's reproducible, correctness-gated benchmark\nharness — both an internal engineering suite and a public, customer-runnable\ndiagnostic. Every number it emits ships with the dataset seed, the workload\nprofile, and a correctness verdict.

\n
\n

Policy: WeftDB benchmarks guide real engineering decisions, not synthetic wins.\nA latency number is only publishable when its correctness check passes.

\n
\n

What it does today:

\n
    \n
  • Seeded workload profiles — the flagship interpolation-heavy-irregular\nprofile generates an irregularly-spaced, gap-containing series\n(ChaCha8Rng, published seed → byte-for-byte reproducible). The underlying\nanalytic signal is shape-selectable (MultiSine | Sawtooth | Step |\nDampedSine) and doubles as the accuracy ground truth. By default each sample keeps\nits float's exact binary expansion (~50 significant digits), so a zero-noise sample sits\nexactly on the ground truth; --decimals N rounds samples to N places instead, which is\nwhat throughput runs should use: on 100k samples (86,401-point output, 20 reps, PASS,\n2026-10-09) the exact-expansion series interpolates in p50 51.0 ms against 25.5 ms at\ntwo decimals, because splimes converts every input to f64 and long decimals take\nbigdecimal's slow conversion path (weft-bench --synthetic --points 100000 --decimals <2|full> --no-gpu-calibrate --reps 20; three runs each, 50.4–53.7 vs 24.6–25.8 ms).
    • \n
  • Storage, read & aggregation workloads — beside the flagship interpolation\nrun, four parallel workloads exercise WeftDB's own hot paths, each with its own\ncorrectness gate and p50/p95/p99 latency: point_lookup (--point-lookup)\nseals a .weftseg segment and times the streaming point read\n(read_segment_point/read_segment_points, single or batch, single-block or\npaged via --pl-rows-per-page, regular closed-form vs irregular), gated against\nthe full-decode value — also over a real corpus with --pl-csv (1M real BTC closes:\na 128-instant batch at p50 2.43 ms single-block vs 0.90 ms paged at 8,192 rows/page,\nweft-bench --point-lookup --pl-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --pl-rows 1000000 --pl-queries 128 [--pl-rows-per-page 8192] --reps 50);\nrange_fetch (--range-fetch, real corpus via --rf-csv: 32 × 100-row windows over 1M\nreal BTC closes at p50 60.8 ms single-block vs 17.8 ms paged) times the windowed\nrange read; compression (--compression) reports realized bytes/point, the\nvalue-column compression ratio, and decode throughput, gated on an exact\nround-trip — over a seeded shape, or a real corpus with --comp-csv <FILE>\n(--csv-value-col, --csv-skip; values read as exact decimal text; --pl-csv does the same for point_lookup). On 1M real\nBTC/USD one-minute closes (rows 3M onward of btc_1min.csv) it measures 4.22 B/point\nrealized (scaled_for, exact), full-decode 20.8M points/sec (p50 52.7 ms, 5 reps),\nand an advisory decimal-exponent FOR footprint of 1.70 B/point\n(weft-bench --compression --comp-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --comp-rows 1000000 --reps 5); and downsample (--downsample) times WeftDB's canonical\nweft-reduce reduction into grid-aligned buckets with a\n--ds-aggs selector over min/max/avg/sum/first/last/p50…p99/twa/twa_linear/twa_bucket_end/sketch_p50…sketch_p99.\n--ds-parallel <N> reduces in N chunks via mergeable partial reductions (identical\nbuckets to serial, asserted by test) — measured 14.7× at 64 chunks on a quiet 16-core box\n(441.2 ms → 30.1 ms, 1,134,659 → 16,921,104 points/sec, --ds-aggs sketch_p99, 500k points,\n5 reps, correctness PASS); a re-run on 2026-10-08 at load average ~19 measured 327.3 → 48.5 ms\n(6.7×).\nThe real-corpus figures in this bullet come from btc_1min.csv, a local file not distributed\nwith the repository (see Ingest); point --pl-csv / --rf-csv / --comp-csv /\n--ds-csv at a CSV of your own to run the same workloads on real data. Do not quote the\ngenerated storage corpora as performance figures: they flatter WeftDB. Measured back to back\non 2026-10-11 at 1M rows (load average ~210): the generators seal at 0.53–0.92 B/point against\n4.22 for real closes, and the reads run faster in proportion: a 128-instant point batch 0.66 vs\n1.58 ms single-block and 0.51 vs 0.75 ms paged, 32 × 100-row windows 17.7 vs 48.1 ms and 9.0 vs\n14.8 ms. Full-decode throughput is the exception (~22M points/sec for every shape and the real\nseries). The downsample generator errs the other way (see above).\nThe underlying point/range read speedups are quantified at the codec layer in\nweft-physical-type/benches/pointread.rs.
    • \n
  • Gap fill (--gap-fill) — the shape of TSM-Bench's interpolation query Q5\n(SAMPLE BY … FILL(LINEAR)): a seeded series loses whole outages (runs of buckets,\n--gf-outage percent of them, --gf-outage-len buckets on average), and the timed query\nreduces what is left and fills every empty bucket with weft_reduce::fill\n(--gf-fill linear|prev|null|<decimal>). It is gated on a dense grid whose synthesized\n(count == 0) buckets are exactly the outages, and it scores the filled values against\nthe mean of the generator's noise-free signal under each outage bucket. One day of one-second\nsamples into minute buckets (70,020 surviving samples, 1,440 buckets, 273 filled; 10 reps,\nPASS, 2026-10-09, load average ~15): linear p50 4.70 ms, RMSE 0.87 (max error 2.54);\nprev 3.73 ms, RMSE 6.36; the constant 50 4.69 ms, RMSE 13.72; null 3.51 ms (nothing to\nscore). At 1M generated samples (809,920 surviving, 16,667 buckets) a linear fill takes\n62.4 ms against 63.5 ms for null, so the fill itself is lost in the reduction's noise\n(weft-bench --gap-fill [--gf-points 1000000] --gf-fill <M> --reps 10). --gf-csv <FILE> cuts\nthe same outages from a real series and scores the fills against the real values removed;\nthe series' own empty buckets are filled too but not scored. On 1M real BTC/USD one-minute\ncloses (2017-09-14 to 2019-08-10, mean $7,336; hourly buckets, 3,168 of 16,667 hours cut;\n10 reps, PASS): linear p50 57.2 ms, MAE $49.27 (0.67% of the mean), RMSE $91.77, worst\n$957.18; prev MAE $76.45, RMSE $147.23; null 55.1 ms (--gf-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --gf-points 1000000 --gf-bucket h; the local corpus\ndescribed under Ingest, not distributed with the repository). --gf-fill cubic /\nquadratic fill with a splimes spline through the bucket values instead (Fill::Spline).\nThat is not a general win, and the default stays linear: on the smooth generated signal a\ncubic fill cuts RMSE 7.5× (0.115 vs 0.865), but on real BTC closes, which move close to a\nrandom walk, linear is more accurate — hourly MAE $49.27 linear vs $55.34 cubic vs $61.32\nquadratic, and at one-minute buckets (187,421 filled minutes, outages averaging three) $6.79\nvs $8.37 — while the spline costs ~14% more time (64.8 vs 56.7 ms hourly; 10 reps,\n2026-10-09, load average ~6). Reach for the spline on smooth physical signals, not on prices.\nWeftDB's grid has only unit widths, so Q5's literal 5-second buckets cannot be expressed yet.
    • \n
  • Vendor-neutral adapters — every system is driven through the\nSystemAdapter trait: the WeftDB reference adapter (splimes' Interpolator on Backend::Auto),\na precision-aware portable linear baseline (fair-protocol class C), and a\nforward-fill/LOCF baseline (class B — the in-process mirror of\nFILL(previous) / locf()).
    • \n
  • Quality beside speed — reconstructions are scored against the analytic\nground truth: RMSE / MAE / max-error / bias. Honest finding, surfaced not\nhidden: on smooth multisine (and step) WeftDB's cubic leads on RMSE, but on\nsawtooth the portable linear baseline out-accuracies the cubic, which\novershoots sharp discontinuities.
    • \n
  • Fair-protocol statistics — p50/p95/p99 + min/max/mean/stddev with\nseeded bootstrap confidence intervals, and an end-to-end timing breakdown\nproving the latency distribution times only the adapter call, never setup.
    • \n
  • Storage estimate — each result carries a StorageEstimate (recommended\nencoding + value/timestamp/total bytes/point) computed by\nweft-physical-type over the actual columns. It also surfaces advisory\npotential-saving estimates for codecs not yet realized on disk: the best-of\nf64 value codec (Gorilla / Chimp / Chimp128 XOR, advisory_best_f64_bytes /\n_codec) for a lossy F64 column, and the Sprintz FIRE forecaster's\nfootprint on the timestamp column (advisory_fire_timestamp_bytes), and the\ndecimal-exponent FOR value codec (advisory_dfor_value_bytes, schema v17,\npresent only when it beats the realized codec — e.g. a ScaledI64 column whose\nscale is forced by a few high-precision values; real BTC closes measure 13.58 vs\n33.74 bits/value in\nweft-physical-type/benches/alp_vs_f64_codecs.rs),\nplus two timestamp-column what-ifs (schema v18), each present only when it beats the\nrealized codec: advisory_common_multiple_timestamp_bytes (the GCD of the deltas\nfactored out — millisecond-precise instants stored in microseconds measure 20 → 10\nbits/value in\nweft-physical-type/benches/timestamp_vs_pco.rs)\nand advisory_delta_for_timestamp_bytes (first-order deltas under per-block FOR, for\nindependent random intervals: 20.0 → 19.4 bits/value)\n— each a what-if number the adopt-or-drop decision reads, never a realized\nheadline claim. The f64 codecs and FIRE live behind weft-physical-type's\nexperimental-codecs feature, which only weft-bench enables.
    • \n
  • Reports — a BenchReport JSON artifact (run metadata + a best-effort\nhardware probe: CPU model, cores, RAM, and the kind/file system/mount of the disk under the working directory) under reports/json/, plus a\nself-contained HTML view (--html) with the most-accurate row highlighted, and a flat\nParquet results table (--parquet, same stem): one row per result with the run metadata\nrepeated (machine, GPU and driver, workload, latency percentiles, throughput, timing,\ncorrectness, nullable accuracy and storage columns), so a directory of runs reads as one\ndataset in DuckDB, Polars or Grafana.
    • \n
  • GPU memory stability (--gpu-memory, needs a hardware GPU) — rounds of GPU\ninterpolations (Backend::Gpu, cubic, f64) at mixed output sizes (--gm-sizes, default\n16 Ki, 1 Mi and 4 Mi points, seeded order), sampling splimes' buffer-pool statistics after\nevery call; gated on full finite grids, idle pool bytes never over the cap, and no idle-byte\ngrowth after the first round. The report's gpu_pool block (schema v22) records the cap and\nthe buffer sets created, reused and evicted. On the RTX 4070 Ti SUPER (driver 610.57.04; 20\nrounds, 60 calls, 2026-10-09): with splimes' default 512 MiB pool, 2 buffer sets created in\nall, none after the first round, 58 reuses, 0 evictions, idle bytes steady at 112 MiB, p50\n8.29 ms per call, 152M output points/sec; with --gm-pool-mib 32, 20 evictions and 19\nre-creations (the 4 Mi-point set churns every round) at an unchanged p50 of 7.96 ms, so on\nthis GPU eviction churn costs nothing measurable next to the kernels.
    • \n
  • First rep apart from the rest — every result with two or more reps records its first\ntimed rep separately from the warm ones (cold_warm, schema v21; printed as first rep\nand in the Parquet table), while latency still covers every rep. It is first-in-process,\nnot OS-cold (the data is already in memory). Measured 2026-10-09 (release, default knobs\nunless noted): interpolation (100k two-decimal samples) 1.00× the warm p50, downsample (500k,\nhourly avg) 0.99×, range fetch 1.04×, gap fill 1.13×, point lookup 1.30×, and\ncompression 4.93× (1.198 vs 0.243 ms), so that workload's p95 over 10 reps is its first\ndecode.
    • \n
  • The engine the server runs — an interpolation run first calls\nsplimes::calibrate() once, as weft-server does at startup (skipping a\nCPU/software adapter the same way), so Backend::Auto uses rayon and the GPU\nwhere they are faster on this machine. The calibration, GPU, its driver version and\nthe thresholds are printed and recorded in the report's metadata.engine (schema v16;\ngpu_driver since v20);\n--no-gpu-calibrate skips it and records splimes' defaults. There is no\ncounterpart to the server's WEFT_GPU_CALIBRATE=force: on a software adapter the\nbench always skips calibration, so there it cannot reproduce a server started with\nforce.
    • \n
  • ILP / TSBS input — a .lp / TSBS file drives the same harness, correctness\ngate, and reporting via the shared weft-line-protocol parser.
    • \n
\n
# Benchmark a line-protocol file, WeftDB vs the portable baselines, JSON + HTML report:\ncargo run -p weft-bench -- \\\n    --input data.lp --field usage --spline cubic --resolution minutes --compare --html\n\n# Seeded synthetic mode — the only mode with a known ground truth, so it also scores accuracy:\ncargo run -p weft-bench -- \\\n    --synthetic --points 300 --noise 0 --shape sawtooth \\\n    --spline cubic --resolution minutes --reps 5 --compare\n\n# Storage/read/aggregation workloads over WeftDB's own hot paths:\ncargo run -p weft-bench -- --point-lookup --pl-rows 100000 --pl-queries 128 --reps 50\ncargo run -p weft-bench -- --range-fetch --rf-window 100 --rf-windows 32 --reps 30\ncargo run -p weft-bench -- --compression --comp-shape clustered --reps 20\ncargo run --release -p weft-bench -- --compression --comp-csv database/datasets/btc_1min.csv --csv-value-col 4 --csv-skip 3000000 --comp-rows 1000000 --reps 5\ncargo run -p weft-bench -- --downsample --ds-bucket m --ds-aggs min,max,p99,twa --reps 20

The process exits non-zero when any result's correctness gate fails, so scripted\ncallers can gate on it. External-engine competitor adapters (DuckDB, ClickHouse,\nInfluxDB 3, QuestDB, TimescaleDB) are the next tracked step in\nROADMAP.md.

\n
\n

Development

\n
# Run the full test suite:\ncargo test\n\n# Skip long-running tests:\nSKIP_SLOW_TESTS=1 cargo test          # PowerShell: $env:SKIP_SLOW_TESTS=1; cargo test\n\n# Test a single crate:\ncargo test -p weftdb

The workspace ships extensive Criterion benchmark\nsuites alongside the Weft-Bench harness (see\nBenchmark Harness):

\n\n\n\n\n\n\n\n\n\n\n\n
CrateBenchmarks
weftdbinterpolation_benchmarks, integration_interpolation_benchmarks, optimized_interpolation_benchmarks, strategy_selection_benchmarks, new_api_benchmarks, cache_performance_benchmarks
\n
cargo bench -p weftdb --bench cache_performance_benchmarks

These benches create their databases in a temporary data dir of their own, which they\nremove when they finish, never in ~/.weftdb/data. That dir is under the system temp\ndir, often a tmpfs; to bench another filesystem, set TEST_DATA_DIR to a directory on\nit (the benches remove the databases they create there, not the directory).

\n

HTML reports are generated under target/criterion/. The interpolation engine's own\nbenchmarks live in the splimes repository.

\n

Formatting uses nightly-only rustfmt options (imports_granularity,\ngroup_imports), so the style gate runs on nightly even though the crates\nthemselves build on stable:

\n
cargo +nightly fmt --all
\n

Project layout

\n
WeftDB/\n├── Cargo.toml                  # workspace manifest (members + shared deps)\n├── weftdb/                     # time-series database (Turso/libSQL control plane + segment store)\n│   └── src/types/\n│       ├── database/           # connection, config, inputs, outputs, pipeline\n│       ├── compression/        # tiered dataset compression\n│       ├── batches/            # batching & analysis\n│       └── …                   # patterns, events, correlations, signals, …\n├── weft-orchestration/         # pipeline orchestration & event detectors\n├── weft-physical-type/         # physical encodings, schemas, timestamp codecs, .weftseg\n├── weft-arrow/                 # Arrow/Parquet interchange for segments\n├── weft-arrow-store/           # Arrow/Parquet bridge over the segment store\n├── weft-line-protocol/         # InfluxDB Line Protocol parser (shared dialect)\n├── weft-reduce/                # downsampling reductions over a bucket grid\n├── weft-server/                # axum HTTP API server\n├── weft-bench/                 # benchmark harness (adapters, profiles, reports)\n└── weft-tui/                   # terminal UI binary

The complete work queue and design constraints live in\nROADMAP.md — the single source of truth.

\n
\n

Crates

\n

WeftDB is a Rust Cargo workspace, from a low-level numerical engine up to an HTTP\nserver and an interactive application:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
CrateRole
splimes (own repo, from crates.io)Spline interpolation engine — Linear / Quadratic / Cubic / Polynomial methods on the CPU (one thread or rayon) or the GPU, chosen by grid size, with raw / interpolated / extrapolated provenance on every point.
weftdbTime-series database: Turso (libSQL) control plane (catalog, metadata, segment index; MVCC concurrent writes) + WeftDB's own typed columnar .weftseg segment store on the measurement hot path, plus the pattern-recognition types and tiered dataset compression. Interpolate-on-read over stored segments (SegmentStore::interpolate_range): a splimes grid reconstructed straight from an aspect's persisted samples (index prune → .weftseg decode → knots, with a margin of extra knots past each edge → the caller's Interpolator, any backend), with raw/interpolated/extrapolated provenance. interpolate_range_f64 returns f64 values instead: splimes computes in f64 either way, so only raw points (exact in the BigDecimal result) differ, and it skips building a BigDecimal per output point, which is most of the cost. Not yet on the HTTP API. Measured end to end (weftdb/benches/interpolate_range.rs, 1 Mi stored irregular samples in 16 segments, cubic to one-second points, two runs, 2026-10-09, load average ~11): ~1M output points from 100k knots in 14.1–16.3 ms with f64 output on rayon against 43.3–51.5 ms with BigDecimal output; the GPU (RTX 4070 Ti SUPER) takes 17.0–29.8 ms (f64) / 53.3–66.4 ms, one thread 42.6–48.8 / 132.8–149.7 ms; at ~100k points rayon takes 3.0 ms (f64) / 7.1 ms. Of the BigDecimal call on rayon, read and decode are ~4 ms, the kernel ~8 ms and the per-point BigDecimal output ~30 ms. The GPU is not the fastest backend on this path.
weft-orchestrationHigh-level pipeline that chains batching → pattern extraction → event detection → correlation → signal generation, with built-in detectors and parallel execution.
weft-physical-typeVendor-neutral physical type system — schema-declared numeric encodings with explicit exactness, timestamp codecs, and the .weftseg columnar segment format (single-block and paged).
weft-arrow / weft-arrow-storeApache Arrow / Parquet interchange for sealed segments and stored reads, kept in leaf crates so the arrow-* dependency tree never reaches the hot-path core.
weft-line-protocolDependency-free InfluxDB Line Protocol parser shared by the server and the benchmark harness.
weft-reduceVendor-neutral downsampling reductions, computable over parts and merged (reduce_partial/PartialReduction, exact for every reduction) — min/max/avg/sum/first/last + nearest-rank p50/p90/p95/p99 + three time-weighted averages (LOCF, linear/trapezoidal, LOCF-to-bucket-end) + mergeable bounded-error sketch_p* percentiles, over an epoch-aligned bucket grid, computed in BigDecimal. A PartialReduction is serde-serializable (so a segment's partial can be persisted and merged later, in place of re-reading it) and re-bucketable to any coarser nesting resolution (rebucket/grids_nest). Gap filling (fill): the dense bucket grid between two bounds, measured buckets unchanged and every empty step synthesized with count == 0 by a declared Fill (Null, Previous, Linear by grid step, or a constant, SQL FILL(…)'s vocabulary, plus Spline, a splimes spline through the bucket values). Shared by the HTTP downsample endpoint and the benchmark harness.
weft-serverBenchmark-grade axum HTTP API — interpolation, downsampling, storage ingest/query, catalog management, Prometheus metrics, live latency profiles.
weft-benchReproducible, correctness-gated benchmark harness — the roadmap's spine; both an internal suite and a customer-runnable diagnostic.
weft-tuiTerminal user interface (Ratatui + Crossterm) for creating databases, importing CSV data, browsing and plotting aspects, and running compression.
\n

The platform is designed for real-time and large-scale workloads: measurements\nare stored with BigDecimal logical precision,\ninterpolation scales from a handful of points to millions, from one CPU thread to the\nrayon pool and, where calibration measured it faster, the GPU, and the storage layer\nuses bulk transactions, cached connections, and typed columnar segments throughout.

\n
\n

Tuning tips

\n
    \n
  • Batch your writes. batch_capture_measurements uses bulk transactions and is\ndramatically faster than single inserts.
    • \n
  • Connections are cached. The database layer reuses connections through a\nTTL/LRU cache (default: 50 connections, 30-minute TTL) and starts MVCC\nBEGIN CONCURRENT transactions automatically, with retry + backoff on\ntransient failures.
    • \n
  • Calibrate the backends once. weft-server does it in the background at startup; a program\nembedding the libraries calls splimes::calibrate() (or splimes::prewarm_gpu()\nplus splimes::set_auto_thresholds) once, off any latency-critical path. Without\nit, interpolation never uses the GPU.
    • \n
  • Pick sensible batch sizes for pipelines (typically 24–100 for hourly data).
    • \n
  • Let the engine choose. Backend::Auto (behind analyze_range and every\nendpoint) already selects the backend by grid size — overriding is rarely necessary.
    • \n
  • Pipeline pattern loading is memory-aware: batch sizes adapt to available RAM to\navoid exhaustion on large datasets.
    • \n
\n
\n

Roadmap

\n

ROADMAP.md is the single source of truth for everything planned,\nin progress, and done — a phase-based [ ]/[x] checkbox queue organized around\nthe benchmark-led commercial thesis (north star: dollars per billion\ninterpolated output points at a p95 latency target). Shipped capabilities are\ndocumented here as features; every performance claim links to a benchmark\nartifact.

\n
\n

License

\n

Copyright (c) 2025-2026 Justin Icenhour.

\n

Licensed under either of

\n\n

at your option.

\n

Two crates include code adapted from Apache-2.0 projects — the Chimp codecs in\nweft-physical-type and the DDSketch quantile sketch in weft-reduce. Those portions\nstay under the Apache License 2.0 whichever option you choose. NOTICE lists\nthem, and each crate's THIRD-PARTY-NOTICES file carries the attribution and the\nlicense text.

\n

Contribution

\n

Unless you explicitly state otherwise, any contribution intentionally submitted for\ninclusion in the work by you, as defined in the Apache-2.0 license, shall be dual\nlicensed as above, without any additional terms or conditions.

\n

Every pull request also needs either a DCO sign-off on each commit or a signed CLA; see\nContribution terms.

\n", "releases": [ { "tag": "v0.1.0", @@ -479,7 +479,7 @@ }, "Nanna": { "repo": "Nanna", - "fetchedAt": "2026-10-13T05:41:37.706Z", + "fetchedAt": "2026-10-14T04:35:18.446Z", "description": "A high-performance, always-on personal AI presence written in Rust — local-first, runs on your own machine with a small open model on a single consumer GPU", "htmlUrl": "https://github.com/basic-automation/Nanna", "homepage": null, @@ -576,7 +576,7 @@ }, "splimes": { "repo": "splimes", - "fetchedAt": "2026-10-13T05:41:33.126Z", + "fetchedAt": "2026-10-14T04:35:13.043Z", "description": "Spline interpolation over irregularly sampled time series, on SIMD/parallel CPU or GPU (wgpu).", "htmlUrl": "https://github.com/basic-automation/splimes", "homepage": "https://docs.rs/splimes", @@ -636,7 +636,7 @@ }, "download": null, "crateVersion": "1.2.0", - "crateDownloads": 277, + "crateDownloads": 299, "crateUrl": "https://crates.io/crates/splimes", "docsUrl": "https://docs.rs/splimes" } From fb656bbfc3ef94864c6fdf062ee85a8353d800a5 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Wed, 14 Oct 2026 00:01:35 -0500 Subject: [PATCH 12/13] fix(checks): a stall says whether the page still runs anything Caught one more check:navigation stall tonight with the new report: on Skidbladnir at 1280 px, the first page of the pass in a fresh tab, on arrive(blog) right after clicking the blog tab, renderer idle (0.00 CPU s in 1 s). That rules out the longest README and an old tab, and says the page was waiting, not computing. The report now also tries a plain evaluate (visibilityState, readyState, path) with 5 s to answer: on a stand-in that spins, no answer; on an idle one, "visible, complete, /projects/y/about". The next real stall will say whether only the stuck promise's timers were held or the page was blocked. scripts/check-navigation.mjs, ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 23 +++++++++++++++++------ scripts/check-navigation.mjs | 11 ++++++++++- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 50ca338..1e3054b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1262,17 +1262,24 @@ record. There is no run log. request now fails after 180 s, naming itself — and on 2026-10-13 it did, in 1 of 3 Chromium runs: `arrive`'s evaluate on Nanna at 1280 px went 180 s unanswered, though its own loop gives up after - 8 s. Both stalls were on Nanna — the longest README, 25,000 px at - phone width — and both on the last page of their width's pass, - where the tab is oldest (~7 min). A stalled page no longer ends + 8 s. Then once more in four runs, with the diagnostics below: on + Skidbladnir at 1280 px — the *first* page of the pass, in a fresh + tab, so not Nanna's length and not an old tab — on `arrive(blog)` + right after the click on the blog tab, with the renderer **idle** + (0.00 CPU s in 1 s). An idle page whose 8-second timers never fire + is waiting on something, not computing. A stalled page no longer ends the run: it fails with the last four calls made of it and, in Chromium, the busiest renderer's CPU time over a second (`busy()` in `scripts/lib/browser.mjs`, from `SystemInfo.getProcessInfo`), and the run goes on in a fresh tab. Shown on a stand-in page that spins after `load`: "renderer busy (0.99 CPU s in 1 s)", and the - next page checked. Next: the busy/idle answer from the next real - stall — busy points at the page (layout or script over Nanna's - README), idle at timers held back in an old tab. + next page checked. The report now also says whether a plain + evaluate still answers while the stuck one waits (stand-in: + spinning, "no answer in 5 s"; idle, "answered: visible, complete, + /projects/y/about"). Next: that answer from the next real stall — + it answers: the event loop turns and only that promise's timers + are held (throttling or a frozen-then-thawed page); it does not: + the page is blocked (a dialog, a synchronous wait). - [ ] One reload of splimes' open README at 1280 px landed 31 px past the reader (4,170 → 4,201) in that same run against main's build. The trace shows the fold restoring 4,201 — the position @@ -1302,6 +1309,10 @@ record. There is no run log. other browser without the chain gets (`CSS.supports('-moz-appearance', 'none')` is true in Firefox 155, false in Chromium 152). Look before and after; Tor Browser is Firefox 140 ESR and gets the same answer. + Or, so the routine can look for itself: a headless Wayland compositor + on the host (`cage` or `sway`, a `pacman` install — **owner**) would + let it run a real, composited Firefox off-screen, away from the + desktop session. - [x] No dead controls without script. Every page is server-rendered and diff --git a/scripts/check-navigation.mjs b/scripts/check-navigation.mjs index b7e131d..97f1edb 100644 --- a/scripts/check-navigation.mjs +++ b/scripts/check-navigation.mjs @@ -304,7 +304,16 @@ try { // end of the run. Say where it was and whether its renderer was busy // or idle, then carry on in a fresh tab. const busy = await browser.busy().catch(() => null) - const where = `${/got no answer/.test(err.message) ? 'stalled' : 'failed'} after ${calls.join(' → ')}${busy === null ? '' : `; renderer ${busy > 0.5 ? 'busy' : 'idle'} (${busy.toFixed(2)} CPU s in 1 s)`}` + // Whether the page still runs a plain script while the stuck one + // waits: if it does, its event loop turns and only that promise's + // timers are held back; if not, the page itself is blocked. + const answers = await Promise.race([ + page.evaluate(`document.visibilityState + ', ' + document.readyState + ', ' + location.pathname`).catch((e) => `error: ${e.message.slice(0, 60)}`), + new Promise((r) => setTimeout(() => r(null), 5000)), + ]) + const where = `${/got no answer/.test(err.message) ? 'stalled' : 'failed'} after ${calls.join(' → ')}` + + (busy === null ? '' : `; renderer ${busy > 0.5 ? 'busy' : 'idle'} (${busy.toFixed(2)} CPU s in 1 s)`) + + `; a plain evaluate ${answers === null ? 'got no answer in 5 s' : `answered: ${answers}`}` console.log(` ✗ ${name.padEnd(7)} /projects/${slug}/about`) const message = err.message.split('\n')[0] console.error(` ${message}\n ${where}`) From 1dea8ae4f04afddb64e399871b8ff5a51f500fbb Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Wed, 14 Oct 2026 00:42:19 -0500 Subject: [PATCH 13/13] docs(roadmap): tonight's run counts for the reload drift and the stall 15 complete Chromium check:navigation runs on this branch without the splimes reload drift; the stall recurred once in the seven hunt runs. ROADMAP.md Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 1e3054b..ae1fe86 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1262,7 +1262,7 @@ record. There is no run log. request now fails after 180 s, naming itself — and on 2026-10-13 it did, in 1 of 3 Chromium runs: `arrive`'s evaluate on Nanna at 1280 px went 180 s unanswered, though its own loop gives up after - 8 s. Then once more in four runs, with the diagnostics below: on + 8 s. Then once more in seven, with the diagnostics below: on Skidbladnir at 1280 px — the *first* page of the pass, in a fresh tab, so not Nanna's length and not an old tab — on `arrive(blog)` right after the click on the blog tab, with the renderer **idle** @@ -1285,7 +1285,7 @@ record. There is no run log. The trace shows the fold restoring 4,201 — the position `pagehide` saved — so the page had moved by 31 px between the check's last reading and the unload. Seen once; not seen on this - branch's build in four runs. Not the unsized shields.io badges + branch's build in 15 Chromium runs. Not the unsized shields.io badges that night's local server served (fixed below): they move the page 2.3 px, not 31. Next: whether anything else above the reader still moves it after `arrive` settles.