Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions crates/devtools/src/planner_fuzz/relations/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -909,6 +909,7 @@ fn with_operator(node: &OperatorNode, operator: Operator) -> Rc<OperatorNode> {
/// estimate can stand in for it without changing what the nodes above
/// compute. Returns its state and statistic.
fn root_estimate(root: &Rc<OperatorNode>) -> Result<(Rc<OperatorNode>, SketchStatistic), String> {
let root = &column_projections_folded(root);
let estimate = match root.non_asap() {
Some(NonASAPOp::Project { cols, child, .. })
if cols
Expand All @@ -931,6 +932,52 @@ fn root_estimate(root: &Rc<OperatorNode>) -> Result<(Rc<OperatorNode>, SketchSta
}
}

/// `root` with a chain of `Project`s of plain column references folded
/// into one: SQL puts a grouped estimate under two (`Project [city, v]`
/// over `Project [v, city]` over the `SummaryEstimate`), and the outer one
/// picks columns of the inner one's output. A root that is not such a
/// chain is returned as it is.
fn column_projections_folded(root: &Rc<OperatorNode>) -> Rc<OperatorNode> {
let plain_columns = |cols: &[ProjectItem]| {
cols.iter()
.map(|item| match item.expr {
ScalarExpr::Column(index) => Some(index),
_ => None,
})
.collect::<Option<Vec<usize>>>()
};
let Some(NonASAPOp::Project { cols, child }) = root.non_asap() else {
return root.clone();
};
let child = column_projections_folded(child);
let (
Some(outer),
Some(NonASAPOp::Project {
cols: inner_cols,
child: grandchild,
}),
) = (plain_columns(cols), child.non_asap())
else {
return root.clone();
};
let Some(inner) = plain_columns(inner_cols) else {
return root.clone();
};
let folded = outer
.iter()
.zip(cols)
.map(|(&index, item)| ProjectItem {
alias: item.alias.clone(),
expr: ScalarExpr::Column(inner[index]),
})
.collect();
OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project {
cols: folded,
child: grandchild.clone(),
}))
.unwrap_or_else(|_| root.clone())
}

/// `root` with its root estimate ([`root_estimate`]) replaced by
/// `estimate`; a root `Project` keeps its columns by name, except the
/// `dropped` group labels (a rollup's), which it loses. Any other column
Expand All @@ -940,6 +987,7 @@ fn with_root_estimate(
estimate: Rc<OperatorNode>,
dropped: &[String],
) -> Result<Rc<OperatorNode>, String> {
let root = &column_projections_folded(root);
let Some(NonASAPOp::Project { cols, child }) = root.non_asap() else {
return Ok(estimate);
};
Expand Down
50 changes: 39 additions & 11 deletions crates/devtools/tests/planner_fuzz.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,20 +219,48 @@ fn witness_of(relation: Relation, workload: FuzzWorkload) -> Witness {
}

/// The sketch rollup witness builds and costs for a grouped quantile:
/// a `SummaryMerge` with `group_by` over the finer per-group KLL.
/// a `SummaryMerge` with `group_by` over the finer per-group KLL. The
/// query must select a KLL, or there is no sketch to roll up. A quantile
/// selects a sketch only over many rows per group and evaluation: the
/// generated PromQL `quantile by (job, instance)` reads 40 latest samples
/// per group, and #902's p90 per city over 5 min at 666.67 rows/s, so
/// both plan exact. p90 per (city, host) over 1 h at 100,000 rows/s
/// (4 cities x 5 hosts, ε 0.05, δ 0.01) reads a KLL tumbling-window
/// chain (k = 132), and p90 per city rolls up from it.
#[test]
fn sketch_rollup_witness_costs_for_a_grouped_quantile() {
let q = promql(
"m_gauge",
None,
Some((OuterOp::Quantile(0.9), &["job", "instance"])),
);
assert_eq!(
witness_of(Relation::CoarserGrouping, generated(q, true)),
Witness::SketchRollup {
labels: vec!["job".into()]
use asap_types::accuracy::AccuracyTarget;
let mut workload = seed("issue902.json");
let per_city_and_host = SqlSpec {
table: "metrics".into(),
aggregate: SqlAggregate::Quantile(0.9),
group_by: vec!["city".into(), "host".into()],
filters: vec![],
window_ms: 3_600_000,
};
workload.queries = vec![workload.queries[2]
.with_spec(QuerySpec::Sql(per_city_and_host))
.with_accuracy(AccuracyTarget::max_error(0.05).failure_probability(0.01))];
workload.data = serde_json::from_value(serde_json::json!({
"arrival": {"ingesting": {"rows_per_second": 100_000.0}},
"sources": {"metrics": {
"series_count": 100,
"distinct_values_per_label": {"city": 4, "host": 5},
}},
}))
.expect("the data facts deserialize");
match check(Relation::CoarserGrouping, workload, 0) {
Outcome::Holds(m) | Outcome::Violated(m) => {
assert!(m.base_plan.contains("Kll"), "{}", m.base_plan);
assert_eq!(
m.witness.and_then(|w| w.derivation),
Some(Witness::SketchRollup {
labels: vec!["city".into()]
})
);
}
);
Outcome::Inconclusive { reason } => panic!("CoarserGrouping is inconclusive: {reason}"),
}
}

/// The exact rollup witness builds and costs for a grouped count (a sum
Expand Down
31 changes: 31 additions & 0 deletions crates/devtools/tests/sketch_plan_snapshots/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,37 @@
"covers": "random-mixed seed-862-long workload-01, query 3: DDSketch pane chain",
"workload": "crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json",
"deployment": "mvp"
},
{
"id": "sketch-heavy-univmon-l2-mvp",
"covers": "sketch-heavy UnivMon: l2_over_time over 4 value streams at 10 Hz, 6 h, relative 0.05: an ingestion-time UnivMon tumbling-window chain (5 rows x 2,367 columns)",
"workload": "tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-6h-every60s-releps0.05-d0.05.json",
"deployment": "mvp"
},
{
"id": "sketch-heavy-cmsheap-topk10-mvp",
"covers": "sketch-heavy Count-Min + heap: topk(10) of a rate over 1,000 URLs at L1-normalized 0.01: a query-time CMS heap (depth 10, width 272) over the exact rate chain",
"workload": "tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json",
"deployment": "mvp"
},
{
"id": "sketch-heavy-csheap-topk10-mvp",
"covers": "sketch-heavy Count Sketch + heap: topk(10) of a rate over 300 URLs at L2-normalized 0.05: a query-time Count Sketch heap (depth 117, width 1,200)",
"workload": "tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-1hz-5m-every60s-l2eps0.05-d0.1.json",
"deployment": "mvp"
},
{
"id": "sketch-heavy-sql-distinct-by-region-mvp",
"covers": "sketch-heavy KMV: SQL COUNT(DISTINCT user_id) by region at 100,000 rows/s over 1 h: a KMV tumbling-window chain per region (k = 26,656)",
"workload": "tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__100000rps-3600s-every60s-releps0.05-d0.05.json",
"tables": "tools/dag-viewer/workloads/sketch-heavy/tables.json",
"deployment": "mvp"
},
{
"id": "sketch-heavy-edge-dashboard-mvp",
"covers": "sketch-heavy mixed: top-k, total rate, L2 and distinct over two metrics: CMS heap, exact rate, UnivMon chain (624 columns) and KMV chain in one plan",
"workload": "tools/dag-viewer/workloads/sketch-heavy/mixed-edge-dashboard__u1000-i1-1hz-r4-10hz.json",
"deployment": "mvp"
}
]
}
Loading
Loading