Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 31 additions & 15 deletions crates/integration-tests/tests/accuracy_at_query_roots.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,13 @@ use asap_types::plan::physical_asap_dag::PhysicalASAPDAG;
use asap_types::summary_type::SketchParams;
use asap_types::workload::PlanningWorkload;

/// The series of `latency` in the module's example.
const LATENCY_SERIES: usize = 1_000;

/// PromQL `queries`, each `(query, accuracy JSON)` every 10 s, over
/// `latency` (1,000 series, one sample per second, ingested, values
/// declared non-negative).
fn workload(queries: &[(&str, &str)]) -> PlanningWorkload {
/// `latency` (`series_count` series, one sample per second each,
/// ingested, values declared non-negative).
fn workload(queries: &[(&str, &str)], series_count: usize) -> PlanningWorkload {
let entries: Vec<String> = queries
.iter()
.map(|(query, accuracy)| {
Expand All @@ -24,24 +27,32 @@ fn workload(queries: &[(&str, &str)]) -> PlanningWorkload {
.collect();
serde_json::from_str(&format!(
r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}},
"data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}},
"sources": {{"latency": {{"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000,
"data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": {series_count}.0}}}},
"sources": {{"latency": {{"series_count": {series_count}, "metric_type": "gauge", "sample_interval": 1000,
"column_signs": {{"value": "non_negative"}}}}}}}}}}"#,
entries.join(",")
))
.unwrap()
}

/// The plan Stage 3 selects for `queries`, no raw data kept (every query
/// reads summaries built at ingestion), with accuracy enforced or
/// ignored.
fn plan(queries: &[(&str, &str)], ignore_accuracy: bool) -> PhysicalASAPDAG {
selection(queries, ignore_accuracy).0
/// The plan Stage 3 selects for `queries` over `series_count` series of
/// `latency`, no raw data kept (every query reads summaries built at
/// ingestion), with accuracy enforced or ignored.
pub(crate) fn plan(
queries: &[(&str, &str)],
series_count: usize,
ignore_accuracy: bool,
) -> PhysicalASAPDAG {
selection(queries, series_count, ignore_accuracy).0
}

/// [`plan`], with the accepted accuracy Stage 3 reports per query.
fn selection(queries: &[(&str, &str)], ignore_accuracy: bool) -> (PhysicalASAPDAG, Vec<String>) {
let workload = workload(queries);
fn selection(
queries: &[(&str, &str)],
series_count: usize,
ignore_accuracy: bool,
) -> (PhysicalASAPDAG, Vec<String>) {
let workload = workload(queries, series_count);
let entries: Vec<_> = workload.query_workload.entries().collect();
let data = workload.data_workload.clone().unwrap();
let capabilities = DeploymentCapabilities {
Expand Down Expand Up @@ -89,7 +100,7 @@ fn sketch_parameters(plan: &PhysicalASAPDAG) -> Vec<SketchParams> {
/// not.
#[test]
fn each_query_meets_its_target_at_its_root() {
let (plan, accepted) = selection(&PROBLEM_EXAMPLE, false);
let (plan, accepted) = selection(&PROBLEM_EXAMPLE, LATENCY_SERIES, false);
assert_eq!(
accepted,
["accepted rank (Kll)", "accepted relative (DDSketch)"],
Expand Down Expand Up @@ -140,8 +151,11 @@ fn ignoring_accuracy_plans_without_k() {
.collect::<std::collections::HashSet<_>>()
.len()
};
assert_eq!(algorithms(&plan(&PROBLEM_EXAMPLE, false)), 2);
let (ignored, accepted) = selection(&PROBLEM_EXAMPLE, true);
assert_eq!(
algorithms(&plan(&PROBLEM_EXAMPLE, LATENCY_SERIES, false)),
2
);
let (ignored, accepted) = selection(&PROBLEM_EXAMPLE, LATENCY_SERIES, true);
assert_eq!(algorithms(&ignored), 1, "{:?}", sketch_parameters(&ignored));
assert!(
accepted
Expand All @@ -162,6 +176,7 @@ fn a_whole_result_failure_probability_is_divided_over_the_values() {
"quantile_over_time(0.95, latency[1h])",
r#"{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01}"#,
)],
LATENCY_SERIES,
false,
);
let parameters = sketch_parameters(&plan);
Expand Down Expand Up @@ -191,6 +206,7 @@ fn a_ratio_sharing_its_readouts_with_its_operands_still_meets_its_target() {
target,
),
],
LATENCY_SERIES,
false,
);
let alphas: Vec<f64> = sketch_parameters(&plan)
Expand Down
1 change: 1 addition & 0 deletions crates/integration-tests/tests/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ mod operator_sharing;
mod pass1_candidates;
mod pass2_promql;
mod pass2_sql;
mod per_query_accuracy_targets;
mod per_series_exact_execution;
mod plan_sharing;
mod production_rollup_workloads;
Expand Down
240 changes: 240 additions & 0 deletions crates/integration-tests/tests/per_query_accuracy_targets.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
//! Each query meets its own accuracy target when several queries read one
//! sketch readout group at different targets (`accuracy-models.md`,
//! "Checking each query's target in selection: constraint (K)").
//!
//! Pass 2 gives a readout group read by several queries one readout
//! candidate per requirement level (`readout_requirements.rs`): for
//! `quantile_over_time(0.99, latency[1h])` asked at rank 0.05 and at rank
//! 0.001, one readout requires 0.05 and its copy 0.001, each reading only
//! the KLL states that meet its level. Both queries root at the same group,
//! so only Stage 3's per-query root check of (K) ties the 0.001 query to
//! the 0.001 copy: without it, the plan reads one KLL with k = 52 for both.
//!
//! The check here is independent of the planner's: from each query's root
//! in the selected plan, every sketch readout on its path, with the θ of
//! the state it reads, must have a theoretical guarantee within that
//! query's target.

use asap_accuracy::{theoretical_guarantee, tolerated_limit};
use std::collections::HashSet;

use asap_types::accuracy::AccuracyTarget;
use asap_types::ir::schema::FieldDataType;
use asap_types::ir::{ASAPOp, Operator};
use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId};
use asap_types::stages::ValueSign;
use asap_types::summary_type::SketchParams;

/// The plan the production stages select for `queries` over one series
/// of `latency`, no raw data kept (every query reads summaries built at
/// ingestion).
fn plan(queries: &[(&str, &str)]) -> PhysicalASAPDAG {
crate::accuracy_at_query_roots::plan(queries, 1, false)
}

/// The one sketch readout query `query`'s output reads in `plan`, if it
/// meets `target`: the `SummaryEstimate` below the query's root reads a
/// state whose θ has a theoretical guarantee, at the target's failure
/// probability, in a metric the target allows and within its error. One
/// series and one readout per path, so the target's δ is the readout
/// value's. `Err` says which readout misses and why, or that the path
/// has no sketch readout to check.
fn query_meets_its_target(
plan: &PhysicalASAPDAG,
query: usize,
target: &AccuracyTarget,
) -> Result<PhysicalASAPNodeId, String> {
let node = |id: PhysicalASAPNodeId| plan.nodes.iter().find(|n| n.id == id).unwrap();
let mut pending = vec![plan.roots[query]];
let mut visited = HashSet::new();
let mut readouts = Vec::new();
while let Some(id) = pending.pop() {
if !visited.insert(id) {
continue;
}
pending.extend(
plan.edges
.iter()
.filter(|edge| edge.consumer == id)
.map(|edge| edge.producer),
);
let Operator::ASAP(ASAPOp::SummaryEstimate {
summary_input,
query: statistic,
}) = &node(id).payload
else {
continue;
};
readouts.push(id);
let state = node(*summary_input);
let parameters = state
.parameters
.as_ref()
.ok_or(format!("readout {id:?} reads a state with no θ"))?;
let summary_type = state
.output_schema
.fields
.iter()
.map(|field| field.dtype.clone())
.find(|dtype| matches!(dtype, FieldDataType::Sketch(_)))
.expect("a sketch state has a sketch column");
let failure_probability = target.failure_probability.unwrap_or(1.0);
let guarantee = theoretical_guarantee(
&summary_type,
parameters,
statistic,
failure_probability,
Some(ValueSign::NonNegative),
None,
)
.ok_or(format!(
"{parameters:?} has no guarantee at δ = {failure_probability}"
))?;
let error = guarantee.spec.error.evaluate().unwrap();
if !target.metrics().contains(&guarantee.spec.accuracy_metric)
|| error > tolerated_limit(target.max_error)
{
return Err(format!(
"{parameters:?} guarantees {:?} {error}, the target is {target:?}",
guarantee.spec.accuracy_metric
));
}
}
match readouts[..] {
[readout] => Ok(readout),
_ => Err(format!(
"expected one sketch readout on the path, found {readouts:?}"
)),
}
}

/// Plans `queries` and checks each query's own target on the plan;
/// returns the plan and the readout each query reads.
fn assert_each_query_meets_its_target(
queries: &[(&str, &str)],
) -> (PhysicalASAPDAG, Vec<PhysicalASAPNodeId>) {
let plan = plan(queries);
let readouts = queries
.iter()
.enumerate()
.map(|(index, (query, accuracy))| {
let target: AccuracyTarget = serde_json::from_str(accuracy).unwrap();
query_meets_its_target(&plan, index, &target).unwrap_or_else(|reason| {
panic!("query {index} ({query} at {accuracy}) misses its target: {reason}\nqueries: {queries:?}")
})
})
.collect();
(plan, readouts)
}

const P99: &str = "quantile_over_time(0.99, latency[1h])";
const RANK_0_05: &str =
r#"{"accuracy_metric": "rank", "max_error": 0.05, "failure_probability": 0.01}"#;
const RANK_0_001: &str =
r#"{"accuracy_metric": "rank", "max_error": 0.001, "failure_probability": 0.01}"#;

/// The p99 at rank 0.05 and at rank 0.001, in both orders: the plan reads
/// a KLL meeting 0.001 for the strict query (k = 2,863 on the built-in
/// grid; k = 52 meets only 0.05).
#[test]
fn one_query_text_at_two_rank_targets_reads_a_kll_meeting_the_stricter() {
for queries in [
[(P99, RANK_0_05), (P99, RANK_0_001)],
[(P99, RANK_0_001), (P99, RANK_0_05)],
] {
let (plan, _) = assert_each_query_meets_its_target(&queries);
assert!(
plan.nodes
.iter()
.any(|n| matches!(n.parameters, Some(SketchParams::Kll { k }) if k >= 2_863)),
"{:?}",
plan.nodes.iter().map(|n| &n.parameters).collect::<Vec<_>>()
);
}
}

/// The L2 norm of one series over 24 h at relative 0.1 and at relative
/// 0.05: the plan reads a UnivMon meeting 0.05 for the strict query
/// (2,367 columns of 5 rows on the built-in grid; 624 meet only 0.1).
#[test]
fn one_query_text_at_two_relative_targets_reads_a_univmon_meeting_the_stricter() {
let l2 = "l2_over_time(latency[24h])";
let loose = r#"{"accuracy_metric": "relative", "max_error": 0.1, "failure_probability": 0.1}"#;
let strict =
r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.1}"#;
for queries in [[(l2, loose), (l2, strict)], [(l2, strict), (l2, loose)]] {
let (plan, _) = assert_each_query_meets_its_target(&queries);
assert!(
plan.nodes.iter().any(|n| matches!(
n.parameters,
Some(SketchParams::UnivMon { sketch_cols, .. }) if sketch_cols >= 2_367
)),
"{:?}",
plan.nodes.iter().map(|n| &n.parameters).collect::<Vec<_>>()
);
}
}

/// Two query texts sharing the p99's readout group: the bare p99 at rank
/// 0.05 and `sort` of it at rank 0.001. Their roots differ, but both
/// read the one readout group, whose copies carry both levels.
#[test]
fn two_query_texts_sharing_a_readout_group_each_meet_their_target() {
let sorted_p99 = "sort(quantile_over_time(0.99, latency[1h]))";
for queries in [
[(P99, RANK_0_05), (sorted_p99, RANK_0_001)],
[(sorted_p99, RANK_0_05), (P99, RANK_0_001)],
] {
let (plan, readouts) = assert_each_query_meets_its_target(&queries);
assert_ne!(plan.roots[0], plan.roots[1], "the two texts root apart");
assert_eq!(readouts[0], readouts[1], "both read one readout");
}
}

/// Property: for any set of queries over shared readout groups, each with
/// its own target, every query's own check holds on the selected plan.
/// The sets are drawn with a fixed-seed generator from queries reading the
/// p99 (KLL for rank, DDSketch for relative), the p50 and the L2 norm, at
/// targets from loose to strict.
#[test]
fn every_query_meets_its_own_target_on_the_selected_plan() {
const POOL: [(&str, &str); 9] = [
(P99, RANK_0_05),
(P99, RANK_0_001),
(
P99,
r#"{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01}"#,
),
("sort(quantile_over_time(0.99, latency[1h]))", RANK_0_001),
(
P99,
r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01}"#,
),
(
P99,
r#"{"accuracy_metric": "relative", "max_error": 0.01, "failure_probability": 0.01}"#,
),
("quantile_over_time(0.5, latency[1h])", RANK_0_05),
(
"l2_over_time(latency[1h])",
r#"{"accuracy_metric": "relative", "max_error": 0.1, "failure_probability": 0.1}"#,
),
(
"l2_over_time(latency[1h])",
r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.1}"#,
),
];
// A 64-bit linear congruential generator (Knuth's MMIX constants).
let mut state: u64 = 0x5eed;
let mut next = |bound: usize| {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
((state >> 33) % bound as u64) as usize
};
for _ in 0..16 {
let size = 2 + next(2);
let queries: Vec<(&str, &str)> = (0..size).map(|_| POOL[next(POOL.len())]).collect();
assert_each_query_meets_its_target(&queries);
}
}
Loading