Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion .github/workflows/website.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ on:
branches: [main]
paths:
- "website/**"
- "ui/**"
- ".github/workflows/website.yml"
pull_request:
paths:
- "website/**"
- "ui/**"
- ".github/workflows/website.yml"
workflow_dispatch:

Expand All @@ -33,14 +35,24 @@ jobs:
with:
node-version: 22
cache: npm
cache-dependency-path: website/package-lock.json
cache-dependency-path: |
website/package-lock.json
ui/package-lock.json

- name: Install dependencies
run: npm ci

- name: Build
run: npm run build

# The click-through demo at /demo: the web UI replaying a recording of DuckShop on sqb dev.
- name: Build the UI demo
working-directory: ui
run: |
npm ci
SQB_UI_BASE=/demo VITE_SQB_DEMO=1 npm run build
node scripts/demo-pages.mjs build demo/recording.json.gz ../website/dist/demo

- name: Check links
run: python3 scripts/check_links.py dist

Expand Down
19 changes: 19 additions & 0 deletions ui/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,22 @@ regenerate the extract; `check:command-table` fails when it is stale.
npm run gen:command-table
npm run check:command-table
```

## Static demo

`sqlbuild.com/demo` is this UI with no server behind it: the server client replays
`demo/recording.json.gz`, a recording of DuckShop on `sqb dev`, and refuses every write.
The website workflow builds it:

```sh
SQB_UI_BASE=/demo VITE_SQB_DEMO=1 npm run build
node scripts/demo-pages.mjs build demo/recording.json.gz ../website/dist/demo
```

To record it again, run DuckShop under `sqb dev`, click through the pages with the
browser saving a HAR, then ask the server for every read the pages made:

```sh
node scripts/record-demo.mjs demo/recording.json.gz --server http://127.0.0.1:<port> \
--replace <project path>=/home/demo/duckshop --replace <location name>=duckshop session.har
```
Binary file added ui/demo/recording.json.gz
Binary file not shown.
68 changes: 68 additions & 0 deletions ui/scripts/demo-pages.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// Lays out the static demo for a host without an SPA fallback, such as GitHub Pages.
//
// node scripts/demo-pages.mjs <ui build> <recording.json.gz> <destination>
//
// Copies the build built with SQB_UI_BASE=/demo VITE_SQB_DEMO=1 and the recording, then writes
// the app page as `<page>.html` for every page and recorded item, so an address opened directly
// or reloaded loads the app instead of the site's 404 page.

import { cpSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
import { dirname, join, relative } from 'node:path';
import { gunzipSync } from 'node:zlib';
import { fileURLToPath } from 'node:url';

const [build, recordingPath, destination] = process.argv.slice(2);
if (!build || !recordingPath || !destination) {
console.error('usage: node scripts/demo-pages.mjs <ui build> <recording.json.gz> <destination>');
process.exit(2);
}

/** Pages without a parameter, read from the routes, such as `runs` or `admin/audit`. */
function staticPages(directory, prefix = '') {
return readdirSync(directory).flatMap((name) => {
const path = join(directory, name);
if (!statSync(path).isDirectory() || name.startsWith('[')) return [];
const page = prefix ? `${prefix}/${name}` : name;
const here = readdirSync(path).includes('+page.svelte') ? [page] : [];
return [...here, ...staticPages(path, page)];
});
}

/** Item pages, from the lists the recording answers: every job, run and trigger the demo links to. */
const ITEM_LISTS = [
[/\/targets\/[^/]+\/jobs$/, 'jobs', (item) => item.job_key],
[/\/targets\/[^/]+\/runs\?/, 'runs', (item) => item.run_id],
[/\/targets\/[^/]+\/triggers$/, 'triggers', (item) => item.trigger_id],
];
/** Catalog and check pages only for the items the recording holds, since there are hundreds. */
const RECORDED_NODE = /\/targets\/[^/]+\/nodes\/([^/?]+)\/sql$/;
const RECORDED_CHECK = /\/targets\/[^/]+\/checks\/([^/?]+)$/;

const recording = JSON.parse(gunzipSync(readFileSync(recordingPath)).toString('utf8'));
const items = new Set();
for (const { url, text } of recording.responses) {
const node = url.match(RECORDED_NODE);
if (node) items.add(`catalog/${decodeURIComponent(node[1])}`);
const check = url.match(RECORDED_CHECK);
if (check) items.add(`checks/${decodeURIComponent(check[1])}`);
for (const [pattern, page, key] of ITEM_LISTS) {
if (!pattern.test(url)) continue;
const list = JSON.parse(text);
for (const item of Array.isArray(list) ? list : []) if (key(item)) items.add(`${page}/${key(item)}`);
}
}

const routes = join(dirname(fileURLToPath(import.meta.url)), '../src/routes');
mkdirSync(destination, { recursive: true });
cpSync(build, destination, { recursive: true });
cpSync(recordingPath, join(destination, 'demo-recording.json.gz'));
const app = readFileSync(join(build, 'index.html'));
const pages = [...staticPages(routes).filter((page) => page !== 'login'), ...items];
// Hosts differ on whether `/runs` reads `runs.html` or `runs/index.html`, so both are written.
for (const page of pages) {
for (const file of [join(destination, `${page}.html`), join(destination, page, 'index.html')]) {
mkdirSync(dirname(file), { recursive: true });
writeFileSync(file, app);
}
}
console.log(`Wrote the demo to ${relative(process.cwd(), destination) || '.'}: ${pages.length} pages.`);
88 changes: 88 additions & 0 deletions ui/scripts/record-demo.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Records the static demo from a click-through of the UI on `sqb dev`.
//
// node scripts/record-demo.mjs <recording.json.gz> --server <url> [--replace FROM=TO]... <session.har>...
//
// The browser's HARs name the requests the pages make; each API read is asked again of the
// `sqb dev` server at --server, because a HAR often drops response bodies, and so are reads sent
// as POST, such as plan previews. `--replace` rewrites text everywhere, for example the project path.

import { readFileSync, writeFileSync } from 'node:fs';
import { gzipSync } from 'node:zlib';

const READ_POSTS = [/\/plans$/, /\/runs\/adhoc\/check$/];
// The location's change feed is a live stream; a trigger's change history is an ordinary read.
const SKIPPED = [/^\/api\/auth\//, /\/targets\/[^/]+\/changes$/, /^\/api\/openapi\.json$/];
const AUDIT = '/api/control/audit';
const USAGE = 'usage: node scripts/record-demo.mjs <recording.json.gz> --server <url> [--replace FROM=TO]... <session.har>...';

const [outPath, ...options] = process.argv.slice(2);
let server = null;
const replacements = [];
const hars = [];
for (let i = 0; i < options.length; i++) {
const value = options[i + 1];
if (options[i] === '--server' && value) server = options[++i].replace(/\/$/, '');
else if (options[i] === '--replace' && value?.includes('=')) replacements.push([value.slice(0, value.indexOf('=')), options[++i].slice(value.indexOf('=') + 1)]);
else if (!options[i].startsWith('--')) hars.push(options[i]);
else server = undefined;
}
if (!outPath || !server || !hars.length) {
console.error(USAGE);
process.exit(2);
}
function keep(entry) {
kept.set(JSON.stringify([entry.method, entry.url, entry.body ?? null]), entry);
}
const scrub = (text) => replacements.reduce((out, [from, to]) => out.split(from).join(to), text);

const kept = new Map();
let auditLimit = null;
for (const { request, response } of hars.flatMap((har) => JSON.parse(readFileSync(har, 'utf8')).log.entries)) {
const url = new URL(request.url);
const path = url.pathname + url.search;
if (!url.pathname.startsWith('/api/') || SKIPPED.some((pattern) => pattern.test(url.pathname))) continue;
if (url.pathname === AUDIT) {
auditLimit = url.searchParams.get('limit');
continue;
}
const body = request.postData?.text ? JSON.parse(request.postData.text) : undefined;
let answer;
if (request.method === 'GET') {
const fresh = await fetch(server + path);
answer = { status: fresh.status, contentType: fresh.headers.get('content-type') ?? 'application/json', text: await fresh.text() };
} else if (READ_POSTS.some((pattern) => pattern.test(url.pathname))) {
// A read sent as POST is asked again too; a plan request already taken keeps the HAR's answer.
const fresh = await fetch(server + path, { method: 'POST', headers: { 'content-type': 'application/json' }, body: request.postData?.text });
if (fresh.ok) answer = { status: fresh.status, contentType: fresh.headers.get('content-type') ?? 'application/json', text: await fresh.text() };
else if (response.content?.text) answer = { status: response.status, contentType: response.content.mimeType || 'application/json', text: response.content.text };
else continue;
} else continue;
if (answer.status >= 500) throw new Error(`${server}${path} answered ${answer.status}`);
const entry = {
method: request.method,
url: scrub(path),
...(body === undefined ? {} : { body: JSON.parse(scrub(JSON.stringify(body))) }),
...answer,
text: scrub(answer.text)
};
keep(entry);
}

// The plan previews asked again above add audit entries, so the browser's later audit pages start
// at the wrong entry. The audit log is read last, following its pages from the newest entry.
if (auditLimit) {
for (let before = Number.MAX_SAFE_INTEGER; before; ) {
const path = `${AUDIT}?before=${before}&limit=${auditLimit}`;
const fresh = await fetch(server + path);
const text = await fresh.text();
if (!fresh.ok) throw new Error(`${server}${path} answered ${fresh.status}`);
keep({ method: 'GET', url: path, status: fresh.status, contentType: fresh.headers.get('content-type') ?? 'application/json', text: scrub(text) });
const page = JSON.parse(text);
before = page.length === Number(auditLimit) ? page[page.length - 1].storage_id : 0;
}
}

const responses = [...kept.values()].sort((a, b) => a.url.localeCompare(b.url) || a.method.localeCompare(b.method));
// Run output and statements make the recording several megabytes of text, so it is stored packed.
writeFileSync(outPath, gzipSync(`${JSON.stringify({ recordedAt: new Date().toISOString(), responses })}\n`, { level: 9 }));
console.log(`Recorded ${responses.length} responses to ${outPath}.`);
3 changes: 3 additions & 0 deletions ui/src/lib/api/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import type {
RunRowPage,
RunRowQuery,
RunStatement,
Snapshot,
RunTabCounts,
Scope,
SnapshotChange,
Expand Down Expand Up @@ -134,6 +135,8 @@ export interface ApiClient extends PlatformApi {

listPlatformEvents(query: { activeOnly: boolean }): Promise<PlatformEvent[]>;

/** The newest version deployed to the location; null before its first deploy. */
getLatestSnapshot(scope: Scope): Promise<Snapshot | null>;
/** Null when the location has reported no recent sync. */
getWarehouseSync(scope: Scope): Promise<WarehouseSync | null>;
/** Asks the location to read SQLBuild's state back from the warehouse now. `connection.test` */
Expand Down
22 changes: 17 additions & 5 deletions ui/src/lib/api/http-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-chec
import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toMicrobatchCoverage, toNodeRuns, toColumnLineage } from './http-catalog';
import { auditMatches, toAuditEntry } from './http-audit';
import { toSnapshotChanges } from './http-changes';
import { manualConnectionTest, toLocationDetail, toLocationSummary, toWarehouseConnection, warehouseSyncOf } from './http-locations';
import { manualConnectionTest, snapshotOf, toLocationDetail, toLocationSummary, toWarehouseConnection, warehouseSyncOf } from './http-locations';
import { toAccount, toRoleAssignments, toServerSettings } from './http-people';
import { toMaintenanceHistory } from './http-maintenance';
import { overviewOf } from './http-overview';
Expand All @@ -17,7 +17,7 @@ import { jobDetailOf, runWindowOf, toJob, toTick, toTrigger, toTriggerStateEvent
import { timelineOf } from './http-timeline';
import { RUN_WINDOW, groupRunNodes, parseRunCursor, planResultOf, queueEntriesOf, toAdhocCheck, runDetailOf, runRowPageOver, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs';
import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals';
import type { AuthMode, Bootstrap, Capability, Location, NavCounts, PlatformEvent, Scope, WarehouseSync } from './types';
import type { AuthMode, Bootstrap, Capability, Location, NavCounts, PlatformEvent, Scope, Snapshot, WarehouseSync } from './types';
import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, ServerSettings, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform';
import type { Acknowledgement, CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build';
import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate';
Expand Down Expand Up @@ -79,6 +79,7 @@ const SUPPORTED: ReadonlySet<string> = new Set<keyof ApiClient>([
'platformHealth',
'listRoles',
'getRole',
'getLatestSnapshot',
'getWarehouseSync',
'requestWarehouseSync',
'maintenanceHistory',
Expand Down Expand Up @@ -251,14 +252,18 @@ export type HttpClientOptions = {
baseUrl?: string;
/** Called when the session is missing or rejected, before the request fails. */
onUnauthorized: () => void;
/** Answers the requests instead of the network; the static demo replays a recording through it. */
fetch?: (request: Request) => Promise<Response>;
/** The change feed; by default the server's, followed with `EventSource`. */
changes?: ChangeSubscription;
};

/**
* The server-backed client. Methods without an endpoint throw, and `supports`
* reports them so the shell never shows a page it cannot load.
*/
export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOptions): ApiClient {
const http = createFetch<paths>({ baseUrl, credentials: 'same-origin' });
export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, changes }: HttpClientOptions): ApiClient {
const http = createFetch<paths>({ baseUrl, credentials: 'same-origin', fetch: answer });
// Locations whose bootstrap actions include view.sql, which may hold on only some nodes, and the
// nodes there whose SQL was refused; each refused read is audited, so neither is asked again until reload.
let sqlLocations = new Set<string>();
Expand Down Expand Up @@ -303,7 +308,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt
}
const implemented = {
supports: (method: keyof ApiClient) => SUPPORTED.has(method),
changes: new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)),
changes: changes ?? new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)),
async bootstrap(): Promise<Bootstrap> {
const { data, error, response } = await http.GET('/api/control/bootstrap');
if (response.status === 401) onUnauthorized();
Expand Down Expand Up @@ -543,6 +548,13 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt
}
return found.slice(0, limit);
},
async getLatestSnapshot(scope: Scope): Promise<Snapshot | null> {
const { locationId, target } = parseLocationKey(scope.locationId);
const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/snapshots', { params: { path: { location_id: locationId, target }, query: { limit: 1 } } });
if (response.status === 401) onUnauthorized();
if (!data) throw new ApiError(errorText(error, response), response.status);
return snapshotOf(data[0] ?? null);
},
async getWarehouseSync(scope: Scope): Promise<WarehouseSync | null> {
const { locationId, target } = parseLocationKey(scope.locationId);
const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/health', { params: { path: { location_id: locationId, target } } });
Expand Down
2 changes: 1 addition & 1 deletion ui/src/lib/api/http-locations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ export function manualConnectionTest(record: Wire['LocationWorkRecord'] | null):
};
}

function snapshotOf(published: Wire['PublishedSnapshot'] | null): Snapshot | null {
export function snapshotOf(published: Wire['PublishedSnapshot'] | null): Snapshot | null {
return published
? {
id: published.snapshot_id,
Expand Down
17 changes: 17 additions & 0 deletions ui/src/lib/api/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import type { ApiClient } from './client';
import { createHttpClient } from './http-client';
import { MockApiClient, type MockOptions } from './mock/mock-client';
import { replayFetch, type Recording } from './replay';
import { LocalChangeBus } from './signals';
import { to } from '$lib/paths';
import type { BuiltInRole } from './types';

export type { ApiClient } from './client';
Expand All @@ -13,12 +16,26 @@ const ROLES: readonly BuiltInRole[] = ['viewer', 'operator', 'developer', 'admin
const EVENTS: readonly NonNullable<MockOptions['events']>[] = ['default', 'none', 'all'];
const LIVE: readonly NonNullable<MockOptions['connection']>[] = ['live', 'reconnecting', 'offline'];

/** The static demo (`VITE_SQB_DEMO=1`): the server client replaying a recording served beside the app. */
export const DEMO = import.meta.env.VITE_SQB_DEMO === '1';

async function loadRecording(): Promise<Recording> {
const response = await fetch(to('/demo-recording.json.gz'));
if (!response.ok) throw new Error(`The demo recording is missing (${response.status})`);
const bytes = new Uint8Array(await response.arrayBuffer());
// A host that serves the file with Content-Encoding has already unpacked it.
const packed = bytes[0] === 0x1f && bytes[1] === 0x8b;
const body = packed ? new Response(new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'))) : new Response(bytes);
return body.json();
}

/**
* The client the app uses: the server, or the mock with `?mock` or `VITE_SQB_MOCK=1`.
* With the mock, `?as=<role>`, `?events=none|all` and `?live=reconnecting|offline`
* adjust it for demos.
*/
export function createClient(search: URLSearchParams, onUnauthorized: () => void = () => {}): ApiClient {
if (DEMO) return createHttpClient({ onUnauthorized, fetch: replayFetch(loadRecording), changes: new LocalChangeBus() });
if (!search.has('mock') && import.meta.env.VITE_SQB_MOCK !== '1') return createHttpClient({ onUnauthorized });
const role = search.get('as');
const events = search.get('events');
Expand Down
5 changes: 5 additions & 0 deletions ui/src/lib/api/mock/mock-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import type {
RunOutputLine,
RunStatement,
Scope,
Snapshot,
SnapshotNode,
Timeline,
Trigger,
Expand Down Expand Up @@ -329,6 +330,10 @@ export class MockApiClient implements ApiClient {
return this.#reply(query.activeOnly ? events.filter((e) => e.active) : events);
}

async getLatestSnapshot(scope: Scope): Promise<Snapshot | null> {
return this.#reply(this.#location(scope).snapshot ?? null);
}

async getWarehouseSync(scope: Scope): Promise<WarehouseSync> {
this.#location(scope);
return this.#reply({ ...this.#sync(scope) });
Expand Down
Loading
Loading