From 9f3fcdf2589ec11d1735b0ba10c081e92f138d47 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:56:46 +0000 Subject: [PATCH 1/3] fix(ui): start the UI demo over on a new engine Start over dropped the demo's database, and in the browser ClickHouse drops its tables on threads the engine's worker does not have, so it failed with "Cannot schedule a task" and left the story off its prepared steps: Plan refused every selection and new orders stayed paused. Start over now replaces the engine with a new worker from the download the browser has cached, which also frees the memory dropped tables kept. If the new engine cannot start, queries fail at once instead of waiting on the stopped one. The panel pauses its reads while it starts over and keeps a failed start-over's error instead of clearing it on the next read. The demo's orders table records no landing times, so the Plan page has no retained window for From a time. The demo now says so, and points to a pasted command with --start-time and to sqt playground --template streaming. --- ui/src/lib/demo/_helpers/chdb-engine.ts | 52 ++++++++++++++----- ui/src/lib/demo/_resources/story.resource.ts | 9 +++- .../lib/demo/components/engine-panel.svelte | 13 +++-- ui/src/lib/demo/main/install-demo-fetch.ts | 2 +- ui/src/lib/demo/types.ts | 2 + .../components/plan/replay-window.svelte | 8 +++ .../demo/_resources/story.resource.test.ts | 27 ++++++++++ website/tests/ui-demo.test.mjs | 12 +++++ 8 files changed, 106 insertions(+), 19 deletions(-) diff --git a/ui/src/lib/demo/_helpers/chdb-engine.ts b/ui/src/lib/demo/_helpers/chdb-engine.ts index fefeedde..df08766d 100644 --- a/ui/src/lib/demo/_helpers/chdb-engine.ts +++ b/ui/src/lib/demo/_helpers/chdb-engine.ts @@ -14,6 +14,7 @@ type ChdbModule = { }; type ChdbClient = { query(sql: string, format?: string): Promise<{ text(): string }>; + terminate(): Promise; }; type SizeRow = { name: string; rows: string | number | null; bytes: string | number | null }; @@ -25,14 +26,31 @@ export async function startChdbEngine( database: string, onProgress: (progress: EngineProgress) => void ): Promise { - const wasmUrl: string = await downloadEngine(base, onProgress); - let client: ChdbClient; - try { - const chdb: ChdbModule = await import(/* @vite-ignore */ `${base}index.js`); - client = await chdb.AsyncChdb.create({ moduleUrl: `${base}chdb.mjs`, wasmUrl }); - } finally { - URL.revokeObjectURL(wasmUrl); + /** + * A new engine in its own worker. The browser gives each worker a fixed pool of threads, which + * the default asynchronous file reads and one pipeline thread per core can use up ("Cannot + * schedule a task"); the demo's tables are small, so it reads synchronously on two threads. + */ + async function connect(progress: (progress: EngineProgress) => void): Promise { + const wasmUrl: string = await downloadEngine(base, progress); + let created: ChdbClient; + try { + const chdb: ChdbModule = await import(/* @vite-ignore */ `${base}index.js`); + created = await chdb.AsyncChdb.create({ moduleUrl: `${base}chdb.mjs`, wasmUrl }); + } finally { + URL.revokeObjectURL(wasmUrl); + } + try { + await created.query('SET max_threads = 2'); + await created.query(`SET local_filesystem_read_method = 'pread'`); + } catch (error) { + await created.terminate(); + throw error; + } + return created; } + // Null while a restart has stopped the old engine and the new one has not started. + let client: ChdbClient | null = await connect(onProgress); // The engine deadlocks on overlapping queries, so every caller waits its turn. let queue: Promise = Promise.resolve(); @@ -40,6 +58,8 @@ export async function startChdbEngine( function run(sql: string, format: string = 'TabSeparated'): Promise { const turn: Promise = queue.then(async () => { + // A query sent to a stopped engine would wait forever, so it fails here instead. + if (client === null) throw new Error('The engine stopped while starting over. Use Start over again, or reload the page.'); try { return (await client.query(sql, format)).text(); } catch (error) { @@ -77,16 +97,22 @@ export async function startChdbEngine( const generator: OrderGeneratorResource = createOrderGeneratorResource(addOrder); - // The browser gives the engine a fixed pool of threads. Its default asynchronous file reads and - // one pipeline thread per core can use them all up ("Cannot schedule a task"), and the demo's - // tables are small, so it reads synchronously on two threads. Settings last for the connection. - await run('SET max_threads = 2'); - await run(`SET local_filesystem_read_method = 'pread'`); - const engine: DemoEngine = { database, run, rows, + restart(): Promise { + // Dropping the database instead runs out of the worker's threads, and dropped tables + // stay in memory for minutes; a new worker starts empty from the cached download. + const turn: Promise = queue.then(async () => { + const stopping: ChdbClient | null = client; + client = null; + await stopping?.terminate(); + client = await connect(() => undefined); + }); + queue = turn.catch(() => undefined); + return turn; + }, async tableSizes(): Promise> { const sizes: SizeRow[] = await rows( `SELECT name, total_rows AS rows, total_bytes AS bytes FROM system.tables WHERE database = '${database}'` diff --git a/ui/src/lib/demo/_resources/story.resource.ts b/ui/src/lib/demo/_resources/story.resource.ts index 9ff5538a..5e2235db 100644 --- a/ui/src/lib/demo/_resources/story.resource.ts +++ b/ui/src/lib/demo/_resources/story.resource.ts @@ -18,7 +18,12 @@ import type { * ahead of time (scripts/prepare-demo.mjs); this fills in the values that differ in each tab, * such as deployment IDs and times, and runs the SQL on the engine in order. */ -export function createStoryResource(prepared: PreparedDemo, run: (sql: string) => Promise): DemoStory { +export function createStoryResource( + prepared: PreparedDemo, + run: (sql: string) => Promise, + /** Empties the engine for Start over; the browser replaces it, since dropping uses up its threads. */ + reset: () => Promise = () => run(`DROP DATABASE IF EXISTS ${prepared.database}`) +): DemoStory { const values: SlotValues = { deployments: {}, uuids: {}, @@ -89,7 +94,7 @@ export function createStoryResource(prepared: PreparedDemo, run: (sql: string) = acting = true; stateName = null; try { - await run(`DROP DATABASE IF EXISTS ${prepared.database}`); + await reset(); Object.assign(values, { deployments: {}, uuids: {}, roleStarts: {}, times: {}, startTime: null }); lastBuild = null; await replaySetup(() => undefined); diff --git a/ui/src/lib/demo/components/engine-panel.svelte b/ui/src/lib/demo/components/engine-panel.svelte index 6cd108de..468a1f3d 100644 --- a/ui/src/lib/demo/components/engine-panel.svelte +++ b/ui/src/lib/demo/components/engine-panel.svelte @@ -21,6 +21,8 @@ let error = $state(null); async function refresh(): Promise { + // While starting over the engine is empty, so its tables are missing; the panel waits. + if (restarting) return; const database: string = engine.database; try { // The engine runs queries in the order they arrive, and an action's statements can land @@ -63,11 +65,13 @@ } let restarting = $state(false); + // Kept apart from the reads' error, which the next successful read clears. + let restartError = $state(null); - /** Puts the engine back where the story starts, without downloading it again. */ + /** Puts a new engine where the story starts, from the download the browser has cached. */ async function startOver(): Promise { restarting = true; - error = null; + restartError = null; engine.stopOrders(); try { await live.story.startOver(); @@ -75,7 +79,7 @@ adding = true; await onrestarted(); } catch (caught) { - error = caught instanceof Error ? caught.message : String(caught); + restartError = `Start over failed: ${caught instanceof Error ? caught.message : String(caught)}`; adding = engine.addingOrders; } finally { restarting = false; @@ -127,6 +131,9 @@ {#if error !== null}
{error}
{/if} + {#if restartError !== null} +
{restartError}
+ {/if}
{adding ? 'A new order arrives every 4 s.' : 'New orders are paused.'} diff --git a/ui/src/lib/demo/main/install-demo-fetch.ts b/ui/src/lib/demo/main/install-demo-fetch.ts index 0f32d24a..b77251ea 100644 --- a/ui/src/lib/demo/main/install-demo-fetch.ts +++ b/ui/src/lib/demo/main/install-demo-fetch.ts @@ -22,7 +22,7 @@ export function installDemoFetch(): void { const started: Promise = (async () => { const { default: prepared } = (await import('../prepared.json')) as unknown as { default: PreparedDemo }; const engine: DemoEngine = await startChdbEngine(base, prepared.database, controller.showProgress); - const story: DemoStory = createStoryResource(prepared, engine.run); + const story: DemoStory = createStoryResource(prepared, engine.run, engine.restart); await story.replaySetup((step: string, done: number, total: number) => controller.showProgress({ phase: 'replay', step, done, total }) ); diff --git a/ui/src/lib/demo/types.ts b/ui/src/lib/demo/types.ts index bdfe3d9d..00935400 100644 --- a/ui/src/lib/demo/types.ts +++ b/ui/src/lib/demo/types.ts @@ -109,6 +109,8 @@ export type DemoEngine = { run(sql: string, format?: string): Promise; /** Rows of a query, as objects. */ rows(sql: string): Promise; + /** Replaces the engine with an empty one, after the queries already asked for. */ + restart(): Promise; /** Each table's rows and bytes, as ClickHouse reports them. */ tableSizes(): Promise>; /** Adds an order to the source table every few seconds, as a stream would. */ diff --git a/ui/src/lib/presentation/components/plan/replay-window.svelte b/ui/src/lib/presentation/components/plan/replay-window.svelte index 1c7cfe58..8355907d 100644 --- a/ui/src/lib/presentation/components/plan/replay-window.svelte +++ b/ui/src/lib/presentation/components/plan/replay-window.svelte @@ -241,6 +241,14 @@ Choose at least one model or pipeline to set a start time. SQLTide requires --start-time to be scoped by --select.

+ {:else if !hasRetentionWindow && import.meta.env.VITE_SQLTIDE_DEMO === '1'} + +

+ This playground's orders table doesn't record when each order landed, so there is no retained + window to pick a start time from. Paste a command with --start-time in + Preview a command to replay from a time here, or run + sqt playground --template streaming to try it with ClickHouse on your machine. +

{:else if !hasRetentionWindow}

A retained source window is not available yet. Build the source first, then choose a diff --git a/ui/tests/lib/demo/_resources/story.resource.test.ts b/ui/tests/lib/demo/_resources/story.resource.test.ts index 979cbf85..015f90fe 100644 --- a/ui/tests/lib/demo/_resources/story.resource.test.ts +++ b/ui/tests/lib/demo/_resources/story.resource.test.ts @@ -114,6 +114,33 @@ describe('story', () => { expect(demo.plan(new URLSearchParams(PLAN_QUERY))).toEqual({ status: 409, body: { detail: DEMO_OFF_STORY_MESSAGE } }); }); + it('given the story left its prepared states when started over then the engine is reset, not dropped, and the plan answers again', async () => { + const ran: string[] = []; + let resets: number = 0; + const demo: DemoStory = createStoryResource( + story(), + async (sql: string) => { + ran.push(sql); + }, + async () => { + resets += 1; + } + ); + await demo.replaySetup(() => undefined); + await demo.build({ selectors: ['pipeline:pl__orders'], startTime: '2026-01-01T10:01:00Z', confirmations: [] }); + await flush(); + await demo.deploymentAction('promote', ran[1].split("', '")[1]); + ran.length = 0; + + await demo.startOver(); + + expect(resets).toBe(1); + expect(ran.some((sql) => sql.startsWith('DROP'))).toBe(false); + expect(ran[0]).toMatch(/^CREATE TABLE db\.t__/); + expect(demo.stateName).toBe('nothing-staged'); + expect(demo.plan(new URLSearchParams(PLAN_QUERY)).status).toBe(200); + }); + it('given Execute on the deployment the plan proposed when promoted then that deployment is the one promoted', async () => { const ran: string[] = []; const demo: DemoStory = await started(ran); diff --git a/website/tests/ui-demo.test.mjs b/website/tests/ui-demo.test.mjs index 5cca7f40..0684f408 100644 --- a/website/tests/ui-demo.test.mjs +++ b/website/tests/ui-demo.test.mjs @@ -94,6 +94,8 @@ test('on the real engine Execute, Promote and Roll back run, and every page read await expect(page.getByTestId('demo-unprepared-selector')).toContainText('sqt playground'); await picker.fill('pipeline:pl__orders'); await page.getByRole('option', { name: /pipeline:pl__orders/ }).click(); + // Its source records no landing times, so picking a start time says why and where to try it. + await expect(page.getByTestId('demo-no-replay-window')).toContainText('sqt playground --template streaming'); await page.getByRole('button', { name: 'Execute' }).click({ timeout: 60_000 }); await expect(page).toHaveURL(/\/demo\/runs\//); await expect(page.getByText('succeeded').first()).toBeVisible({ timeout: 120_000 }); @@ -122,6 +124,16 @@ test('on the real engine Execute, Promote and Roll back run, and every page read await sidebar('Runs'); await page.getByRole('button', { name: /^Deployments \d+$/ }).click(); await expect(page.getByText('deployment rollback').first()).toBeVisible(); + + // Start over brings the engine back to the story's start, so the pipeline plans again. + await panel.getByRole('button', { name: 'Start over' }).click(); + await expect(panel.getByRole('button', { name: 'Start over' })).toBeEnabled({ timeout: 120_000 }); + await expect(panel.getByRole('alert')).toHaveCount(0); + await expect(panel.getByText('A new order arrives every 4 s.')).toBeVisible(); + await sidebar('Plan'); + await page.getByRole('combobox').fill('pipeline:pl__orders'); + await page.getByRole('option', { name: /pipeline:pl__orders/ }).click(); + await expect(page.getByRole('button', { name: 'Execute' })).toBeEnabled({ timeout: 60_000 }); expect(errors).toEqual([]); }); From c4f3e7d73be05bf5b7ffce567d38bc84c1cc75e4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:32:03 +0000 Subject: [PATCH 2/3] docs: say rows arriving during a build can be delivered twice A build starts the live materialized view before it captures the inclusive replay boundary, so every row that arrives in between can reach the target through both, not only the row at the boundary. The replay lineage and models pages now say so, and that a plain MergeTree target keeps those duplicates unless the model SQL or the engine handles them. --- website/src/content/docs/docs/concepts/models.mdx | 5 +++-- .../content/docs/docs/concepts/replay-lineage.mdx | 14 +++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/website/src/content/docs/docs/concepts/models.mdx b/website/src/content/docs/docs/concepts/models.mdx index d8886a6d..8493aaa0 100644 --- a/website/src/content/docs/docs/concepts/models.mdx +++ b/website/src/content/docs/docs/concepts/models.mdx @@ -169,5 +169,6 @@ predicates on their input query instead. Common engines include `MergeTree`, `ReplacingMergeTree`, `SummingMergeTree`, and `AggregatingMergeTree`, including replicated variants. Engine choice is part of replay correctness: -the replay cutoff is inclusive, so authored SQL and storage engines must tolerate duplicate delivery -where necessary. +rows that arrive while a build runs can be delivered twice, once by the live view and once by replay +(see [replay lineage](/docs/concepts/replay-lineage/#inclusive-at-least-once-cutoffs)). A plain +`MergeTree` target holds those duplicates unless the model SQL or the engine handles them. diff --git a/website/src/content/docs/docs/concepts/replay-lineage.mdx b/website/src/content/docs/docs/concepts/replay-lineage.mdx index 20bd63e9..d42e9350 100644 --- a/website/src/content/docs/docs/concepts/replay-lineage.mdx +++ b/website/src/content/docs/docs/concepts/replay-lineage.mdx @@ -20,11 +20,15 @@ The normalized columns are `_replay_partition`, `_replay_offset`, `_replay_times ## Inclusive, at-least-once cutoffs -Every replay boundary is inclusive. A row at boundary `B` can reach the target twice: once through -the live materialized view and once through replay. This way no row is missed, but SQLTide does -not remove the duplicate. - -Model SQL and ClickHouse engines own deduplication or aggregate idempotency. +Every replay boundary is inclusive, and a build starts the live materialized view before it +captures the boundary. Every row that arrives between those two steps, not only the row at boundary +`B`, can reach the target twice: once through the live view and once through replay. This way no +row is missed, but SQLTide does not remove the duplicates. The number of duplicated rows grows with +the arrival rate and the time the build waits for the live view to stabilise. For example, with one +order arriving every 20 ms during a build, 135 of 189 orders reached the new target twice. + +A plain `MergeTree` target keeps every duplicate, so its row counts and sums are inflated. Model SQL +or the engine, such as `ReplacingMergeTree` or idempotent aggregates, must handle them. ## Repeated offsets From bd14362775be016f04c79b03bed586f7cdaa33f7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:41:37 +0000 Subject: [PATCH 3/3] build: stamp each release's version into the UI demo The demo's sidebar showed the sqt version that prepared it, so it fell behind with every release. CI already checks that prepared.json matches the current sqt apart from that version, so each release PR now writes its version into preparedWith and the recorded /api/status. --- release-please-config.json | 10 ++++++++++ website/README.md | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/release-please-config.json b/release-please-config.json index 1a1c2b47..157fb873 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -12,6 +12,16 @@ "type": "toml", "path": "Cargo.toml", "jsonpath": "$.workspace.package.version" + }, + { + "type": "json", + "path": "ui/src/lib/demo/prepared.json", + "jsonpath": "$.preparedWith" + }, + { + "type": "json", + "path": "ui/src/lib/demo/recordings.json", + "jsonpath": "$.responses['/api/status'].body.toolVersion" } ], "changelog-sections": [ diff --git a/website/README.md b/website/README.md index c067fd13..17cbe83a 100644 --- a/website/README.md +++ b/website/README.md @@ -34,7 +34,9 @@ isolated, which the engine needs. Refresh both files with `node ui/scripts/prepare-demo.mjs`, which runs the playground with Docker and the installed `sqt`. CI runs it with `--check`, which fails when `prepared.json` no longer -matches what `sqt` does. `tests/ui-demo.test.mjs` opens every recorded page and fails on any +matches what `sqt` does apart from its version. Each release PR writes the new version into +`preparedWith` and the recorded `/api/status`, so the demo shows the release it was built with. +`tests/ui-demo.test.mjs` opens every recorded page and fails on any unrecorded request, runs every prepared action on the real engine, and checks each fallback. The tests serve the build with `scripts/serve.mjs`, which applies `public/_headers` as Pages does.