Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions release-please-config.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@
"type": "toml",
"path": "Cargo.toml",
"jsonpath": "$.workspace.package.version"
},
{
"type": "json",
"path": "ui/src/lib/demo/prepared.json",
"jsonpath": "$.preparedWith"
},
{
"type": "json",
"path": "ui/src/lib/demo/recordings.json",
"jsonpath": "$.responses['/api/status'].body.toolVersion"
}
],
"changelog-sections": [
Expand Down
52 changes: 39 additions & 13 deletions ui/src/lib/demo/_helpers/chdb-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ type ChdbModule = {
};
type ChdbClient = {
query(sql: string, format?: string): Promise<{ text(): string }>;
terminate(): Promise<void>;
};

type SizeRow = { name: string; rows: string | number | null; bytes: string | number | null };
Expand All @@ -25,21 +26,40 @@ export async function startChdbEngine(
database: string,
onProgress: (progress: EngineProgress) => void
): Promise<DemoEngine> {
const wasmUrl: string = await downloadEngine(base, onProgress);
let client: ChdbClient;
try {
const chdb: ChdbModule = await import(/* @vite-ignore */ `${base}index.js`);
client = await chdb.AsyncChdb.create({ moduleUrl: `${base}chdb.mjs`, wasmUrl });
} finally {
URL.revokeObjectURL(wasmUrl);
/**
* A new engine in its own worker. The browser gives each worker a fixed pool of threads, which
* the default asynchronous file reads and one pipeline thread per core can use up ("Cannot
* schedule a task"); the demo's tables are small, so it reads synchronously on two threads.
*/
async function connect(progress: (progress: EngineProgress) => void): Promise<ChdbClient> {
const wasmUrl: string = await downloadEngine(base, progress);
let created: ChdbClient;
try {
const chdb: ChdbModule = await import(/* @vite-ignore */ `${base}index.js`);
created = await chdb.AsyncChdb.create({ moduleUrl: `${base}chdb.mjs`, wasmUrl });
} finally {
URL.revokeObjectURL(wasmUrl);
}
try {
await created.query('SET max_threads = 2');
await created.query(`SET local_filesystem_read_method = 'pread'`);
} catch (error) {
await created.terminate();
throw error;
}
return created;
}
// Null while a restart has stopped the old engine and the new one has not started.
let client: ChdbClient | null = await connect(onProgress);

// The engine deadlocks on overlapping queries, so every caller waits its turn.
let queue: Promise<unknown> = Promise.resolve();
let nextOrder: { id: number; offset: number } = { id: 0, offset: 0 };

function run(sql: string, format: string = 'TabSeparated'): Promise<string> {
const turn: Promise<string> = queue.then(async () => {
// A query sent to a stopped engine would wait forever, so it fails here instead.
if (client === null) throw new Error('The engine stopped while starting over. Use Start over again, or reload the page.');
try {
return (await client.query(sql, format)).text();
} catch (error) {
Expand Down Expand Up @@ -77,16 +97,22 @@ export async function startChdbEngine(

const generator: OrderGeneratorResource = createOrderGeneratorResource(addOrder);

// The browser gives the engine a fixed pool of threads. Its default asynchronous file reads and
// one pipeline thread per core can use them all up ("Cannot schedule a task"), and the demo's
// tables are small, so it reads synchronously on two threads. Settings last for the connection.
await run('SET max_threads = 2');
await run(`SET local_filesystem_read_method = 'pread'`);

const engine: DemoEngine = {
database,
run,
rows,
restart(): Promise<void> {
// Dropping the database instead runs out of the worker's threads, and dropped tables
// stay in memory for minutes; a new worker starts empty from the cached download.
const turn: Promise<void> = queue.then(async () => {
const stopping: ChdbClient | null = client;
client = null;
await stopping?.terminate();
client = await connect(() => undefined);
});
queue = turn.catch(() => undefined);
return turn;
},
async tableSizes(): Promise<Map<string, TableSize>> {
const sizes: SizeRow[] = await rows<SizeRow>(
`SELECT name, total_rows AS rows, total_bytes AS bytes FROM system.tables WHERE database = '${database}'`
Expand Down
9 changes: 7 additions & 2 deletions ui/src/lib/demo/_resources/story.resource.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,12 @@ import type {
* ahead of time (scripts/prepare-demo.mjs); this fills in the values that differ in each tab,
* such as deployment IDs and times, and runs the SQL on the engine in order.
*/
export function createStoryResource(prepared: PreparedDemo, run: (sql: string) => Promise<unknown>): DemoStory {
export function createStoryResource(
prepared: PreparedDemo,
run: (sql: string) => Promise<unknown>,
/** Empties the engine for Start over; the browser replaces it, since dropping uses up its threads. */
reset: () => Promise<unknown> = () => run(`DROP DATABASE IF EXISTS ${prepared.database}`)
): DemoStory {
const values: SlotValues = {
deployments: {},
uuids: {},
Expand Down Expand Up @@ -89,7 +94,7 @@ export function createStoryResource(prepared: PreparedDemo, run: (sql: string) =
acting = true;
stateName = null;
try {
await run(`DROP DATABASE IF EXISTS ${prepared.database}`);
await reset();
Object.assign(values, { deployments: {}, uuids: {}, roleStarts: {}, times: {}, startTime: null });
lastBuild = null;
await replaySetup(() => undefined);
Expand Down
13 changes: 10 additions & 3 deletions ui/src/lib/demo/components/engine-panel.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
let error = $state<string | null>(null);

async function refresh(): Promise<void> {
// While starting over the engine is empty, so its tables are missing; the panel waits.
if (restarting) return;
const database: string = engine.database;
try {
// The engine runs queries in the order they arrive, and an action's statements can land
Expand Down Expand Up @@ -63,19 +65,21 @@
}

let restarting = $state<boolean>(false);
// Kept apart from the reads' error, which the next successful read clears.
let restartError = $state<string | null>(null);

/** Puts the engine back where the story starts, without downloading it again. */
/** Puts a new engine where the story starts, from the download the browser has cached. */
async function startOver(): Promise<void> {
restarting = true;
error = null;
restartError = null;
engine.stopOrders();
try {
await live.story.startOver();
await engine.startOrders();
adding = true;
await onrestarted();
} catch (caught) {
error = caught instanceof Error ? caught.message : String(caught);
restartError = `Start over failed: ${caught instanceof Error ? caught.message : String(caught)}`;
adding = engine.addingOrders;
} finally {
restarting = false;
Expand Down Expand Up @@ -127,6 +131,9 @@
{#if error !== null}
<div style:color="var(--sb-error)">{error}</div>
{/if}
{#if restartError !== null}
<div style:color="var(--sb-error)" role="alert">{restartError}</div>
{/if}
<div class="flex items-center justify-between gap-2 border-t border-border pt-2">
<span class="text-muted-foreground">{adding ? 'A new order arrives every 4 s.' : 'New orders are paused.'}</span>
<button type="button" class="text-primary hover:underline" onclick={toggleOrders}>{adding ? 'Pause' : 'Resume'}</button>
Expand Down
2 changes: 1 addition & 1 deletion ui/src/lib/demo/main/install-demo-fetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export function installDemoFetch(): void {
const started: Promise<LiveDemo> = (async () => {
const { default: prepared } = (await import('../prepared.json')) as unknown as { default: PreparedDemo };
const engine: DemoEngine = await startChdbEngine(base, prepared.database, controller.showProgress);
const story: DemoStory = createStoryResource(prepared, engine.run);
const story: DemoStory = createStoryResource(prepared, engine.run, engine.restart);
await story.replaySetup((step: string, done: number, total: number) =>
controller.showProgress({ phase: 'replay', step, done, total })
);
Expand Down
2 changes: 2 additions & 0 deletions ui/src/lib/demo/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ export type DemoEngine = {
run(sql: string, format?: string): Promise<string>;
/** Rows of a query, as objects. */
rows<Row>(sql: string): Promise<Row[]>;
/** Replaces the engine with an empty one, after the queries already asked for. */
restart(): Promise<void>;
/** Each table's rows and bytes, as ClickHouse reports them. */
tableSizes(): Promise<Map<string, TableSize>>;
/** Adds an order to the source table every few seconds, as a stream would. */
Expand Down
8 changes: 8 additions & 0 deletions ui/src/lib/presentation/components/plan/replay-window.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,14 @@
Choose at least one model or pipeline to set a start time. SQLTide requires
<code class="code">--start-time</code> to be scoped by <code class="code">--select</code>.
</p>
{:else if !hasRetentionWindow && import.meta.env.VITE_SQLTIDE_DEMO === '1'}
<!-- The UI demo's orders table, like the basic playground's, records no landing times. -->
<p class="text-muted-foreground text-[12px]" data-testid="demo-no-replay-window">
This playground's orders table doesn't record when each order landed, so there is no retained
window to pick a start time from. Paste a command with <code class="code">--start-time</code> in
Preview a command to replay from a time here, or run
<code class="code">sqt playground --template streaming</code> to try it with ClickHouse on your machine.
</p>
{:else if !hasRetentionWindow}
<p class="text-muted-foreground text-[12px]">
A retained source window is not available yet. Build the source first, then choose a
Expand Down
27 changes: 27 additions & 0 deletions ui/tests/lib/demo/_resources/story.resource.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,33 @@ describe('story', () => {
expect(demo.plan(new URLSearchParams(PLAN_QUERY))).toEqual({ status: 409, body: { detail: DEMO_OFF_STORY_MESSAGE } });
});

it('given the story left its prepared states when started over then the engine is reset, not dropped, and the plan answers again', async () => {
const ran: string[] = [];
let resets: number = 0;
const demo: DemoStory = createStoryResource(
story(),
async (sql: string) => {
ran.push(sql);
},
async () => {
resets += 1;
}
);
await demo.replaySetup(() => undefined);
await demo.build({ selectors: ['pipeline:pl__orders'], startTime: '2026-01-01T10:01:00Z', confirmations: [] });
await flush();
await demo.deploymentAction('promote', ran[1].split("', '")[1]);
ran.length = 0;

await demo.startOver();

expect(resets).toBe(1);
expect(ran.some((sql) => sql.startsWith('DROP'))).toBe(false);
expect(ran[0]).toMatch(/^CREATE TABLE db\.t__/);
expect(demo.stateName).toBe('nothing-staged');
expect(demo.plan(new URLSearchParams(PLAN_QUERY)).status).toBe(200);
});

it('given Execute on the deployment the plan proposed when promoted then that deployment is the one promoted', async () => {
const ran: string[] = [];
const demo: DemoStory = await started(ran);
Expand Down
4 changes: 3 additions & 1 deletion website/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ isolated, which the engine needs.

Refresh both files with `node ui/scripts/prepare-demo.mjs`, which runs the playground with Docker
and the installed `sqt`. CI runs it with `--check`, which fails when `prepared.json` no longer
matches what `sqt` does. `tests/ui-demo.test.mjs` opens every recorded page and fails on any
matches what `sqt` does apart from its version. Each release PR writes the new version into
`preparedWith` and the recorded `/api/status`, so the demo shows the release it was built with.
`tests/ui-demo.test.mjs` opens every recorded page and fails on any
unrecorded request, runs every prepared action on the real engine, and checks each fallback. The
tests serve the build with `scripts/serve.mjs`, which applies `public/_headers` as Pages does.

Expand Down
5 changes: 3 additions & 2 deletions website/src/content/docs/docs/concepts/models.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -169,5 +169,6 @@ predicates on their input query instead.

Common engines include `MergeTree`, `ReplacingMergeTree`, `SummingMergeTree`, and
`AggregatingMergeTree`, including replicated variants. Engine choice is part of replay correctness:
the replay cutoff is inclusive, so authored SQL and storage engines must tolerate duplicate delivery
where necessary.
rows that arrive while a build runs can be delivered twice, once by the live view and once by replay
(see [replay lineage](/docs/concepts/replay-lineage/#inclusive-at-least-once-cutoffs)). A plain
`MergeTree` target holds those duplicates unless the model SQL or the engine handles them.
14 changes: 9 additions & 5 deletions website/src/content/docs/docs/concepts/replay-lineage.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,15 @@ The normalized columns are `_replay_partition`, `_replay_offset`, `_replay_times

## Inclusive, at-least-once cutoffs

Every replay boundary is inclusive. A row at boundary `B` can reach the target twice: once through
the live materialized view and once through replay. This way no row is missed, but SQLTide does
not remove the duplicate.

Model SQL and ClickHouse engines own deduplication or aggregate idempotency.
Every replay boundary is inclusive, and a build starts the live materialized view before it
captures the boundary. Every row that arrives between those two steps, not only the row at boundary
`B`, can reach the target twice: once through the live view and once through replay. This way no
row is missed, but SQLTide does not remove the duplicates. The number of duplicated rows grows with
the arrival rate and the time the build waits for the live view to stabilise. For example, with one
order arriving every 20 ms during a build, 135 of 189 orders reached the new target twice.

A plain `MergeTree` target keeps every duplicate, so its row counts and sums are inflated. Model SQL
or the engine, such as `ReplacingMergeTree` or idempotent aggregates, must handle them.

## Repeated offsets

Expand Down
12 changes: 12 additions & 0 deletions website/tests/ui-demo.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ test('on the real engine Execute, Promote and Roll back run, and every page read
await expect(page.getByTestId('demo-unprepared-selector')).toContainText('sqt playground');
await picker.fill('pipeline:pl__orders');
await page.getByRole('option', { name: /pipeline:pl__orders/ }).click();
// Its source records no landing times, so picking a start time says why and where to try it.
await expect(page.getByTestId('demo-no-replay-window')).toContainText('sqt playground --template streaming');
await page.getByRole('button', { name: 'Execute' }).click({ timeout: 60_000 });
await expect(page).toHaveURL(/\/demo\/runs\//);
await expect(page.getByText('succeeded').first()).toBeVisible({ timeout: 120_000 });
Expand Down Expand Up @@ -122,6 +124,16 @@ test('on the real engine Execute, Promote and Roll back run, and every page read
await sidebar('Runs');
await page.getByRole('button', { name: /^Deployments \d+$/ }).click();
await expect(page.getByText('deployment rollback').first()).toBeVisible();

// Start over brings the engine back to the story's start, so the pipeline plans again.
await panel.getByRole('button', { name: 'Start over' }).click();
await expect(panel.getByRole('button', { name: 'Start over' })).toBeEnabled({ timeout: 120_000 });
await expect(panel.getByRole('alert')).toHaveCount(0);
await expect(panel.getByText('A new order arrives every 4 s.')).toBeVisible();
await sidebar('Plan');
await page.getByRole('combobox').fill('pipeline:pl__orders');
await page.getByRole('option', { name: /pipeline:pl__orders/ }).click();
await expect(page.getByRole('button', { name: 'Execute' })).toBeEnabled({ timeout: 60_000 });
expect(errors).toEqual([]);
});

Expand Down
Loading