Skip to content

Commit 16e22dc

Browse files
authored
feat(server): estimated Snowflake query cost per run, node and warehouse (#1461)
* wip: record estimated query cost per run, resource and warehouse in the state schema Not yet complete: no UI, server reads, docs or integration tests yet. Rust unit tests for the refresh rules pass; clippy is clean. * wip: serve and show estimated Snowflake cost per run, node and warehouse * fix(server): count only each run's newest cost attempt and pass node keys to cost reads
1 parent 444d0bb commit 16e22dc

79 files changed

Lines changed: 3321 additions & 41 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"record": {
3+
"state_database": null,
4+
"state_schema": "main",
5+
"observation_id": "cost-000",
6+
"run_id": "build-1",
7+
"orchestration_run_id": null,
8+
"attempt": 1,
9+
"resource_type": "model",
10+
"resource_name": "orders",
11+
"warehouse_name": "TRANSFORM_WH",
12+
"warehouse_size": "Small",
13+
"warehouse_type": "STANDARD",
14+
"cluster_number": 1,
15+
"query_count": 2,
16+
"attributed_seconds": "120.000",
17+
"bytes_scanned": 1048576,
18+
"estimated_credits": "0.0667",
19+
"estimated_usd": "0.2000",
20+
"status": "complete",
21+
"statements_expected": 4,
22+
"statements_observed": 4,
23+
"limitations": "[]",
24+
"target_name": "development",
25+
"run_started_at": "2026-01-01T00:00:00Z",
26+
"run_completed_at": "2026-01-01T00:10:00Z",
27+
"observed_at": "2026-01-01T00:15:00Z"
28+
},
29+
"observations": [
30+
{"observation_id": "build-1-run-1", "hours_ago": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_type": "_total", "resource_name": "build-1", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "status": "partial", "estimated_credits": "0.15", "estimated_usd": "0.45", "statements_observed": 2},
31+
{"observation_id": "build-1-orders-1", "hours_ago": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "status": "partial", "estimated_credits": "0.10", "estimated_usd": "0.30", "statements_observed": 2},
32+
{"observation_id": "build-1-payments-1", "hours_ago": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_name": "payments", "status": "partial", "estimated_credits": "0.03", "estimated_usd": "0.09", "statements_observed": 2},
33+
{"observation_id": "build-1-customers-1", "hours_ago": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_name": "customers", "status": "partial", "estimated_credits": "0.05", "estimated_usd": "0.15", "statements_observed": 2},
34+
{"observation_id": "build-1-run-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_type": "_total", "resource_name": "build-1", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "estimated_credits": "0.40", "estimated_usd": "1.20"},
35+
{"observation_id": "build-1-orders-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "estimated_credits": "0.25", "estimated_usd": "0.75"},
36+
{"observation_id": "build-1-setup-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_type": "run", "resource_name": "development", "estimated_credits": "0.05", "estimated_usd": "0.15"},
37+
{"observation_id": "build-1-customers-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_name": "customers", "estimated_credits": "0.10", "estimated_usd": "0.30"},
38+
{"observation_id": "build-2-run-1", "hours_ago": 5, "run_id": "build-2", "resource_type": "_total", "resource_name": "build-2", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "estimated_credits": "0.20", "estimated_usd": null},
39+
{"observation_id": "build-2-orders-1", "hours_ago": 5, "run_id": "build-2", "warehouse_name": "LOAD_WH", "warehouse_size": "X-Small", "estimated_credits": "0.20", "estimated_usd": null},
40+
{"observation_id": "build-3-run-1", "hours_ago": 240, "run_id": "build-3", "resource_type": "_total", "resource_name": "build-3", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "estimated_credits": "5.00", "estimated_usd": "15.00"},
41+
{"observation_id": "build-3-orders-1", "hours_ago": 240, "run_id": "build-3", "estimated_credits": "5.00", "estimated_usd": "15.00"}
42+
]
43+
}

‎clients/typescript/src/schema.d.ts‎

Lines changed: 273 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,22 @@ export interface paths {
418418
patch?: never;
419419
trace?: never;
420420
};
421+
"/api/control/locations/{location_id}/targets/{target}/cost": {
422+
parameters: {
423+
query?: never;
424+
header?: never;
425+
path?: never;
426+
cookie?: never;
427+
};
428+
get: operations["readPersonCostSummary"];
429+
put?: never;
430+
post?: never;
431+
delete?: never;
432+
options?: never;
433+
head?: never;
434+
patch?: never;
435+
trace?: never;
436+
};
421437
"/api/control/locations/{location_id}/targets/{target}/fingerprint": {
422438
parameters: {
423439
query?: never;
@@ -546,6 +562,22 @@ export interface paths {
546562
patch?: never;
547563
trace?: never;
548564
};
565+
"/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/cost": {
566+
parameters: {
567+
query?: never;
568+
header?: never;
569+
path?: never;
570+
cookie?: never;
571+
};
572+
get: operations["listPersonNodeCosts"];
573+
put?: never;
574+
post?: never;
575+
delete?: never;
576+
options?: never;
577+
head?: never;
578+
patch?: never;
579+
trace?: never;
580+
};
549581
"/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/runs": {
550582
parameters: {
551583
query?: never;
@@ -642,6 +674,22 @@ export interface paths {
642674
patch?: never;
643675
trace?: never;
644676
};
677+
"/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/cost": {
678+
parameters: {
679+
query?: never;
680+
header?: never;
681+
path?: never;
682+
cookie?: never;
683+
};
684+
get: operations["readPersonRunCost"];
685+
put?: never;
686+
post?: never;
687+
delete?: never;
688+
options?: never;
689+
head?: never;
690+
patch?: never;
691+
trace?: never;
692+
};
645693
"/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/events": {
646694
parameters: {
647695
query?: never;
@@ -1614,6 +1662,45 @@ export interface components {
16141662
access: components["schemas"]["PersonAccess"];
16151663
login_mode: components["schemas"]["LoginMode"];
16161664
};
1665+
/** @description Estimated query cost of one group, such as a job, a warehouse size or a node. */
1666+
CostGroup: {
1667+
/** Format: double */
1668+
credits: number;
1669+
/** @description The group's name; None for builds no server run started, or a resource without a warehouse. */
1670+
key?: string | null;
1671+
/**
1672+
* Format: int64
1673+
* @description Builds that contributed, counted once each.
1674+
*/
1675+
runs: number;
1676+
/** Format: double */
1677+
seconds: number;
1678+
/**
1679+
* Format: double
1680+
* @description None when any of the group's observations had no configured credit price.
1681+
*/
1682+
usd?: number | null;
1683+
};
1684+
/** @description A target's estimated query cost over a window, from the latest observation of each run. */
1685+
CostSummaryRecord: {
1686+
by_job: components["schemas"]["CostGroup"][];
1687+
/** @description Every node that ran, keyed `<resource_type>:<resource_name>` like catalog node keys. */
1688+
by_node: components["schemas"]["CostGroup"][];
1689+
by_warehouse_size: components["schemas"]["CostGroup"][];
1690+
/** Format: double */
1691+
credits: number;
1692+
/**
1693+
* Format: int64
1694+
* @description Runs whose cost is still pending or partial, which a later build re-reads.
1695+
*/
1696+
incomplete_runs: number;
1697+
/** Format: int64 */
1698+
runs: number;
1699+
/** Format: date-time */
1700+
since: string;
1701+
/** Format: double */
1702+
usd?: number | null;
1703+
};
16171704
/** @description One stretch of the cursor, as raw bound texts. */
16181705
CoverageBatch: {
16191706
end: string;
@@ -2049,6 +2136,23 @@ export interface components {
20492136
MutationResponse: {
20502137
applied: boolean;
20512138
};
2139+
/** @description One build's estimated query cost of one node, across every warehouse it used. */
2140+
NodeCostRecord: {
2141+
/** Format: double */
2142+
credits: number;
2143+
orchestration_run_id?: string | null;
2144+
/** Format: int64 */
2145+
query_count: number;
2146+
run_id: string;
2147+
/** Format: date-time */
2148+
run_started_at: string;
2149+
/** Format: double */
2150+
seconds: number;
2151+
/** @description The collection status of the whole build. */
2152+
status: string;
2153+
/** Format: double */
2154+
usd?: number | null;
2155+
};
20522156
/** @enum {string} */
20532157
NodeEventKind: "success" | "failed" | "recovered";
20542158
/** @enum {string} */
@@ -3301,7 +3405,7 @@ export interface components {
33013405
* @description Catalog observations require full scans; only immutable history supports a watermark.
33023406
* @enum {string}
33033407
*/
3304-
WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "observations" | "catalog" | "statistics";
3408+
WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "observations" | "resource_costs" | "catalog" | "statistics";
33053409
/** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */
33063410
WarehouseJanitorRecord: {
33073411
archive_name: string;
@@ -3363,6 +3467,43 @@ export interface components {
33633467
state_schema: string;
33643468
value: string;
33653469
};
3470+
/** @description One estimated query cost observation of a run, resource and warehouse; refreshes append. */
3471+
WarehouseResourceCostRecord: {
3472+
/** Format: int64 */
3473+
attempt: number;
3474+
attributed_seconds: string;
3475+
/** Format: int64 */
3476+
bytes_scanned: number;
3477+
/** Format: int64 */
3478+
cluster_number?: number | null;
3479+
estimated_credits: string;
3480+
estimated_usd?: string | null;
3481+
limitations: string;
3482+
observation_id: string;
3483+
/** Format: date-time */
3484+
observed_at: string;
3485+
orchestration_run_id?: string | null;
3486+
/** Format: int64 */
3487+
query_count: number;
3488+
resource_name: string;
3489+
resource_type: string;
3490+
/** Format: date-time */
3491+
run_completed_at: string;
3492+
run_id: string;
3493+
/** Format: date-time */
3494+
run_started_at: string;
3495+
state_database?: string | null;
3496+
state_schema: string;
3497+
/** Format: int64 */
3498+
statements_expected: number;
3499+
/** Format: int64 */
3500+
statements_observed: number;
3501+
status: string;
3502+
target_name?: string | null;
3503+
warehouse_name: string;
3504+
warehouse_size: string;
3505+
warehouse_type?: string | null;
3506+
};
33663507
/** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */
33673508
WarehouseStateLocation: {
33683509
database?: string | null;
@@ -4468,6 +4609,47 @@ export interface operations {
44684609
};
44694610
};
44704611
};
4612+
readPersonCostSummary: {
4613+
parameters: {
4614+
query?: {
4615+
/** @description Runs that started in the last this many days; defaults to 7, capped at 90 */
4616+
days?: number;
4617+
};
4618+
header?: never;
4619+
path: {
4620+
location_id: string;
4621+
target: string;
4622+
};
4623+
cookie?: never;
4624+
};
4625+
requestBody?: never;
4626+
responses: {
4627+
/** @description Estimated query cost by job, warehouse size and node, most expensive first */
4628+
200: {
4629+
headers: {
4630+
[name: string]: unknown;
4631+
};
4632+
content: {
4633+
"application/json": components["schemas"]["CostSummaryRecord"];
4634+
};
4635+
};
4636+
/** @description Session rejected */
4637+
401: {
4638+
headers: {
4639+
[name: string]: unknown;
4640+
};
4641+
content?: never;
4642+
};
4643+
403: {
4644+
headers: {
4645+
[name: string]: unknown;
4646+
};
4647+
content: {
4648+
"application/json": components["schemas"]["ApiErrorBody"];
4649+
};
4650+
};
4651+
};
4652+
};
44714653
readPersonFingerprint: {
44724654
parameters: {
44734655
query: {
@@ -4811,6 +4993,50 @@ export interface operations {
48114993
};
48124994
};
48134995
};
4996+
listPersonNodeCosts: {
4997+
parameters: {
4998+
query?: {
4999+
/** @description Builds that started in the last this many days; defaults to 7, capped at 90 */
5000+
days?: number;
5001+
/** @description Maximum builds; defaults to 100, capped at 1000 */
5002+
limit?: number;
5003+
};
5004+
header?: never;
5005+
path: {
5006+
location_id: string;
5007+
target: string;
5008+
node_key: string;
5009+
};
5010+
cookie?: never;
5011+
};
5012+
requestBody?: never;
5013+
responses: {
5014+
/** @description The node's estimated query cost in each build, newest first */
5015+
200: {
5016+
headers: {
5017+
[name: string]: unknown;
5018+
};
5019+
content: {
5020+
"application/json": components["schemas"]["NodeCostRecord"][];
5021+
};
5022+
};
5023+
/** @description Session rejected */
5024+
401: {
5025+
headers: {
5026+
[name: string]: unknown;
5027+
};
5028+
content?: never;
5029+
};
5030+
403: {
5031+
headers: {
5032+
[name: string]: unknown;
5033+
};
5034+
content: {
5035+
"application/json": components["schemas"]["ApiErrorBody"];
5036+
};
5037+
};
5038+
};
5039+
};
48145040
listPersonNodeRuns: {
48155041
parameters: {
48165042
query?: {
@@ -5071,6 +5297,52 @@ export interface operations {
50715297
};
50725298
};
50735299
};
5300+
readPersonRunCost: {
5301+
parameters: {
5302+
query?: never;
5303+
header?: never;
5304+
path: {
5305+
location_id: string;
5306+
target: string;
5307+
run_id: string;
5308+
};
5309+
cookie?: never;
5310+
};
5311+
requestBody?: never;
5312+
responses: {
5313+
/** @description The latest estimated query cost observation of the run and of each resource and warehouse it used; empty when none was synced */
5314+
200: {
5315+
headers: {
5316+
[name: string]: unknown;
5317+
};
5318+
content: {
5319+
"application/json": components["schemas"]["WarehouseResourceCostRecord"][];
5320+
};
5321+
};
5322+
/** @description Session rejected */
5323+
401: {
5324+
headers: {
5325+
[name: string]: unknown;
5326+
};
5327+
content?: never;
5328+
};
5329+
403: {
5330+
headers: {
5331+
[name: string]: unknown;
5332+
};
5333+
content: {
5334+
"application/json": components["schemas"]["ApiErrorBody"];
5335+
};
5336+
};
5337+
/** @description Run not found for a person who may view the whole target */
5338+
404: {
5339+
headers: {
5340+
[name: string]: unknown;
5341+
};
5342+
content?: never;
5343+
};
5344+
};
5345+
};
50745346
listPersonRunEvents: {
50755347
parameters: {
50765348
query?: {

‎crates/sqlbuild-analysis/src/janitor_planning/constants.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use sqlbuild_core::constants::STATE_TABLE_NAMES;
44

55
/// SQLBuild state tables the janitor never acts on.
6-
pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 11] = STATE_TABLE_NAMES;
6+
pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 12] = STATE_TABLE_NAMES;
77
/// Prefix of every strict janitor archive name.
88
pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__";
99
/// Separator between archive name segments.

0 commit comments

Comments
 (0)