diff --git a/release-please-config.json b/release-please-config.json index 1a1c2b47..157fb873 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -12,6 +12,16 @@ "type": "toml", "path": "Cargo.toml", "jsonpath": "$.workspace.package.version" + }, + { + "type": "json", + "path": "ui/src/lib/demo/prepared.json", + "jsonpath": "$.preparedWith" + }, + { + "type": "json", + "path": "ui/src/lib/demo/recordings.json", + "jsonpath": "$.responses['/api/status'].body.toolVersion" } ], "changelog-sections": [ diff --git a/ui/src/lib/demo/_helpers/chdb-engine.ts b/ui/src/lib/demo/_helpers/chdb-engine.ts index fefeedde..df08766d 100644 --- a/ui/src/lib/demo/_helpers/chdb-engine.ts +++ b/ui/src/lib/demo/_helpers/chdb-engine.ts @@ -14,6 +14,7 @@ type ChdbModule = { }; type ChdbClient = { query(sql: string, format?: string): Promise<{ text(): string }>; + terminate(): Promise; }; type SizeRow = { name: string; rows: string | number | null; bytes: string | number | null }; @@ -25,14 +26,31 @@ export async function startChdbEngine( database: string, onProgress: (progress: EngineProgress) => void ): Promise { - const wasmUrl: string = await downloadEngine(base, onProgress); - let client: ChdbClient; - try { - const chdb: ChdbModule = await import(/* @vite-ignore */ `${base}index.js`); - client = await chdb.AsyncChdb.create({ moduleUrl: `${base}chdb.mjs`, wasmUrl }); - } finally { - URL.revokeObjectURL(wasmUrl); + /** + * A new engine in its own worker. The browser gives each worker a fixed pool of threads, which + * the default asynchronous file reads and one pipeline thread per core can use up ("Cannot + * schedule a task"); the demo's tables are small, so it reads synchronously on two threads. + */ + async function connect(progress: (progress: EngineProgress) => void): Promise { + const wasmUrl: string = await downloadEngine(base, progress); + let created: ChdbClient; + try { + const chdb: ChdbModule = await import(/* @vite-ignore */ `${base}index.js`); + created = await chdb.AsyncChdb.create({ moduleUrl: `${base}chdb.mjs`, wasmUrl }); + } finally { + URL.revokeObjectURL(wasmUrl); + } + try { + await created.query('SET max_threads = 2'); + await created.query(`SET local_filesystem_read_method = 'pread'`); + } catch (error) { + await created.terminate(); + throw error; + } + return created; } + // Null while a restart has stopped the old engine and the new one has not started. + let client: ChdbClient | null = await connect(onProgress); // The engine deadlocks on overlapping queries, so every caller waits its turn. let queue: Promise = Promise.resolve(); @@ -40,6 +58,8 @@ export async function startChdbEngine( function run(sql: string, format: string = 'TabSeparated'): Promise { const turn: Promise = queue.then(async () => { + // A query sent to a stopped engine would wait forever, so it fails here instead. + if (client === null) throw new Error('The engine stopped while starting over. Use Start over again, or reload the page.'); try { return (await client.query(sql, format)).text(); } catch (error) { @@ -77,16 +97,22 @@ export async function startChdbEngine( const generator: OrderGeneratorResource = createOrderGeneratorResource(addOrder); - // The browser gives the engine a fixed pool of threads. Its default asynchronous file reads and - // one pipeline thread per core can use them all up ("Cannot schedule a task"), and the demo's - // tables are small, so it reads synchronously on two threads. Settings last for the connection. - await run('SET max_threads = 2'); - await run(`SET local_filesystem_read_method = 'pread'`); - const engine: DemoEngine = { database, run, rows, + restart(): Promise { + // Dropping the database instead runs out of the worker's threads, and dropped tables + // stay in memory for minutes; a new worker starts empty from the cached download. + const turn: Promise = queue.then(async () => { + const stopping: ChdbClient | null = client; + client = null; + await stopping?.terminate(); + client = await connect(() => undefined); + }); + queue = turn.catch(() => undefined); + return turn; + }, async tableSizes(): Promise> { const sizes: SizeRow[] = await rows( `SELECT name, total_rows AS rows, total_bytes AS bytes FROM system.tables WHERE database = '${database}'` diff --git a/ui/src/lib/demo/_resources/story.resource.ts b/ui/src/lib/demo/_resources/story.resource.ts index 9ff5538a..5e2235db 100644 --- a/ui/src/lib/demo/_resources/story.resource.ts +++ b/ui/src/lib/demo/_resources/story.resource.ts @@ -18,7 +18,12 @@ import type { * ahead of time (scripts/prepare-demo.mjs); this fills in the values that differ in each tab, * such as deployment IDs and times, and runs the SQL on the engine in order. */ -export function createStoryResource(prepared: PreparedDemo, run: (sql: string) => Promise): DemoStory { +export function createStoryResource( + prepared: PreparedDemo, + run: (sql: string) => Promise, + /** Empties the engine for Start over; the browser replaces it, since dropping uses up its threads. */ + reset: () => Promise = () => run(`DROP DATABASE IF EXISTS ${prepared.database}`) +): DemoStory { const values: SlotValues = { deployments: {}, uuids: {}, @@ -89,7 +94,7 @@ export function createStoryResource(prepared: PreparedDemo, run: (sql: string) = acting = true; stateName = null; try { - await run(`DROP DATABASE IF EXISTS ${prepared.database}`); + await reset(); Object.assign(values, { deployments: {}, uuids: {}, roleStarts: {}, times: {}, startTime: null }); lastBuild = null; await replaySetup(() => undefined); diff --git a/ui/src/lib/demo/components/engine-panel.svelte b/ui/src/lib/demo/components/engine-panel.svelte index 6cd108de..468a1f3d 100644 --- a/ui/src/lib/demo/components/engine-panel.svelte +++ b/ui/src/lib/demo/components/engine-panel.svelte @@ -21,6 +21,8 @@ let error = $state(null); async function refresh(): Promise { + // While starting over the engine is empty, so its tables are missing; the panel waits. + if (restarting) return; const database: string = engine.database; try { // The engine runs queries in the order they arrive, and an action's statements can land @@ -63,11 +65,13 @@ } let restarting = $state(false); + // Kept apart from the reads' error, which the next successful read clears. + let restartError = $state(null); - /** Puts the engine back where the story starts, without downloading it again. */ + /** Puts a new engine where the story starts, from the download the browser has cached. */ async function startOver(): Promise { restarting = true; - error = null; + restartError = null; engine.stopOrders(); try { await live.story.startOver(); @@ -75,7 +79,7 @@ adding = true; await onrestarted(); } catch (caught) { - error = caught instanceof Error ? caught.message : String(caught); + restartError = `Start over failed: ${caught instanceof Error ? caught.message : String(caught)}`; adding = engine.addingOrders; } finally { restarting = false; @@ -127,6 +131,9 @@ {#if error !== null}
{error}
{/if} + {#if restartError !== null} +
{restartError}
+ {/if}
{adding ? 'A new order arrives every 4 s.' : 'New orders are paused.'} diff --git a/ui/src/lib/demo/main/install-demo-fetch.ts b/ui/src/lib/demo/main/install-demo-fetch.ts index 0f32d24a..b77251ea 100644 --- a/ui/src/lib/demo/main/install-demo-fetch.ts +++ b/ui/src/lib/demo/main/install-demo-fetch.ts @@ -22,7 +22,7 @@ export function installDemoFetch(): void { const started: Promise = (async () => { const { default: prepared } = (await import('../prepared.json')) as unknown as { default: PreparedDemo }; const engine: DemoEngine = await startChdbEngine(base, prepared.database, controller.showProgress); - const story: DemoStory = createStoryResource(prepared, engine.run); + const story: DemoStory = createStoryResource(prepared, engine.run, engine.restart); await story.replaySetup((step: string, done: number, total: number) => controller.showProgress({ phase: 'replay', step, done, total }) ); diff --git a/ui/src/lib/demo/types.ts b/ui/src/lib/demo/types.ts index bdfe3d9d..00935400 100644 --- a/ui/src/lib/demo/types.ts +++ b/ui/src/lib/demo/types.ts @@ -109,6 +109,8 @@ export type DemoEngine = { run(sql: string, format?: string): Promise; /** Rows of a query, as objects. */ rows(sql: string): Promise; + /** Replaces the engine with an empty one, after the queries already asked for. */ + restart(): Promise; /** Each table's rows and bytes, as ClickHouse reports them. */ tableSizes(): Promise>; /** Adds an order to the source table every few seconds, as a stream would. */ diff --git a/ui/src/lib/presentation/components/plan/replay-window.svelte b/ui/src/lib/presentation/components/plan/replay-window.svelte index 1c7cfe58..8355907d 100644 --- a/ui/src/lib/presentation/components/plan/replay-window.svelte +++ b/ui/src/lib/presentation/components/plan/replay-window.svelte @@ -241,6 +241,14 @@ Choose at least one model or pipeline to set a start time. SQLTide requires --start-time to be scoped by --select.

+ {:else if !hasRetentionWindow && import.meta.env.VITE_SQLTIDE_DEMO === '1'} + +

+ This playground's orders table doesn't record when each order landed, so there is no retained + window to pick a start time from. Paste a command with --start-time in + Preview a command to replay from a time here, or run + sqt playground --template streaming to try it with ClickHouse on your machine. +

{:else if !hasRetentionWindow}

A retained source window is not available yet. Build the source first, then choose a diff --git a/ui/tests/lib/demo/_resources/story.resource.test.ts b/ui/tests/lib/demo/_resources/story.resource.test.ts index 979cbf85..015f90fe 100644 --- a/ui/tests/lib/demo/_resources/story.resource.test.ts +++ b/ui/tests/lib/demo/_resources/story.resource.test.ts @@ -114,6 +114,33 @@ describe('story', () => { expect(demo.plan(new URLSearchParams(PLAN_QUERY))).toEqual({ status: 409, body: { detail: DEMO_OFF_STORY_MESSAGE } }); }); + it('given the story left its prepared states when started over then the engine is reset, not dropped, and the plan answers again', async () => { + const ran: string[] = []; + let resets: number = 0; + const demo: DemoStory = createStoryResource( + story(), + async (sql: string) => { + ran.push(sql); + }, + async () => { + resets += 1; + } + ); + await demo.replaySetup(() => undefined); + await demo.build({ selectors: ['pipeline:pl__orders'], startTime: '2026-01-01T10:01:00Z', confirmations: [] }); + await flush(); + await demo.deploymentAction('promote', ran[1].split("', '")[1]); + ran.length = 0; + + await demo.startOver(); + + expect(resets).toBe(1); + expect(ran.some((sql) => sql.startsWith('DROP'))).toBe(false); + expect(ran[0]).toMatch(/^CREATE TABLE db\.t__/); + expect(demo.stateName).toBe('nothing-staged'); + expect(demo.plan(new URLSearchParams(PLAN_QUERY)).status).toBe(200); + }); + it('given Execute on the deployment the plan proposed when promoted then that deployment is the one promoted', async () => { const ran: string[] = []; const demo: DemoStory = await started(ran); diff --git a/website/README.md b/website/README.md index c067fd13..17cbe83a 100644 --- a/website/README.md +++ b/website/README.md @@ -34,7 +34,9 @@ isolated, which the engine needs. Refresh both files with `node ui/scripts/prepare-demo.mjs`, which runs the playground with Docker and the installed `sqt`. CI runs it with `--check`, which fails when `prepared.json` no longer -matches what `sqt` does. `tests/ui-demo.test.mjs` opens every recorded page and fails on any +matches what `sqt` does apart from its version. Each release PR writes the new version into +`preparedWith` and the recorded `/api/status`, so the demo shows the release it was built with. +`tests/ui-demo.test.mjs` opens every recorded page and fails on any unrecorded request, runs every prepared action on the real engine, and checks each fallback. The tests serve the build with `scripts/serve.mjs`, which applies `public/_headers` as Pages does. diff --git a/website/src/content/docs/docs/concepts/models.mdx b/website/src/content/docs/docs/concepts/models.mdx index d8886a6d..8493aaa0 100644 --- a/website/src/content/docs/docs/concepts/models.mdx +++ b/website/src/content/docs/docs/concepts/models.mdx @@ -169,5 +169,6 @@ predicates on their input query instead. Common engines include `MergeTree`, `ReplacingMergeTree`, `SummingMergeTree`, and `AggregatingMergeTree`, including replicated variants. Engine choice is part of replay correctness: -the replay cutoff is inclusive, so authored SQL and storage engines must tolerate duplicate delivery -where necessary. +rows that arrive while a build runs can be delivered twice, once by the live view and once by replay +(see [replay lineage](/docs/concepts/replay-lineage/#inclusive-at-least-once-cutoffs)). A plain +`MergeTree` target holds those duplicates unless the model SQL or the engine handles them. diff --git a/website/src/content/docs/docs/concepts/replay-lineage.mdx b/website/src/content/docs/docs/concepts/replay-lineage.mdx index 20bd63e9..d42e9350 100644 --- a/website/src/content/docs/docs/concepts/replay-lineage.mdx +++ b/website/src/content/docs/docs/concepts/replay-lineage.mdx @@ -20,11 +20,15 @@ The normalized columns are `_replay_partition`, `_replay_offset`, `_replay_times ## Inclusive, at-least-once cutoffs -Every replay boundary is inclusive. A row at boundary `B` can reach the target twice: once through -the live materialized view and once through replay. This way no row is missed, but SQLTide does -not remove the duplicate. - -Model SQL and ClickHouse engines own deduplication or aggregate idempotency. +Every replay boundary is inclusive, and a build starts the live materialized view before it +captures the boundary. Every row that arrives between those two steps, not only the row at boundary +`B`, can reach the target twice: once through the live view and once through replay. This way no +row is missed, but SQLTide does not remove the duplicates. The number of duplicated rows grows with +the arrival rate and the time the build waits for the live view to stabilise. For example, with one +order arriving every 20 ms during a build, 135 of 189 orders reached the new target twice. + +A plain `MergeTree` target keeps every duplicate, so its row counts and sums are inflated. Model SQL +or the engine, such as `ReplacingMergeTree` or idempotent aggregates, must handle them. ## Repeated offsets diff --git a/website/tests/ui-demo.test.mjs b/website/tests/ui-demo.test.mjs index 5cca7f40..0684f408 100644 --- a/website/tests/ui-demo.test.mjs +++ b/website/tests/ui-demo.test.mjs @@ -94,6 +94,8 @@ test('on the real engine Execute, Promote and Roll back run, and every page read await expect(page.getByTestId('demo-unprepared-selector')).toContainText('sqt playground'); await picker.fill('pipeline:pl__orders'); await page.getByRole('option', { name: /pipeline:pl__orders/ }).click(); + // Its source records no landing times, so picking a start time says why and where to try it. + await expect(page.getByTestId('demo-no-replay-window')).toContainText('sqt playground --template streaming'); await page.getByRole('button', { name: 'Execute' }).click({ timeout: 60_000 }); await expect(page).toHaveURL(/\/demo\/runs\//); await expect(page.getByText('succeeded').first()).toBeVisible({ timeout: 120_000 }); @@ -122,6 +124,16 @@ test('on the real engine Execute, Promote and Roll back run, and every page read await sidebar('Runs'); await page.getByRole('button', { name: /^Deployments \d+$/ }).click(); await expect(page.getByText('deployment rollback').first()).toBeVisible(); + + // Start over brings the engine back to the story's start, so the pipeline plans again. + await panel.getByRole('button', { name: 'Start over' }).click(); + await expect(panel.getByRole('button', { name: 'Start over' })).toBeEnabled({ timeout: 120_000 }); + await expect(panel.getByRole('alert')).toHaveCount(0); + await expect(panel.getByText('A new order arrives every 4 s.')).toBeVisible(); + await sidebar('Plan'); + await page.getByRole('combobox').fill('pipeline:pl__orders'); + await page.getByRole('option', { name: /pipeline:pl__orders/ }).click(); + await expect(page.getByRole('button', { name: 'Execute' })).toBeEnabled({ timeout: 60_000 }); expect(errors).toEqual([]); });