Skip to content
Merged
109 changes: 92 additions & 17 deletions crates/accuracy/src/error_contract/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//! output error from its inputs' errors (concepts doc, section 4).

use asap_types::accuracy::AccuracyMetric;
use asap_types::ir::operator::arguments::BinaryOpKind;
use asap_types::ir::operator::arguments::{BinaryOpKind, PromQLVectorSetOpKind};
use asap_types::ir::operator::{AggIntent, Reduction};
use asap_types::ir::scalar::expr_ir::ArithmeticOpKind;
use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue};
Expand All @@ -20,12 +20,18 @@ use asap_types::summary_type::{ExactKind, SummaryInputExpr};
/// (concepts doc, section 4, "Local propagation"):
/// - value-preserving (`Sort`, `TimeRange`, `TimeShift`, `Concat`, a
/// readout, a finalize, a `SummaryMerge`): $\beta = 1$ in any metric;
/// - PromQL `and`, `unless` and `or`, which copy values and choose series
/// by labels only, and `label_replace` / `label_join` of labels
/// ([`set_operator_error_factors`], [`reads_an_estimate`]): $\beta = 1$
/// on each input whose values are copied, 0 on `and`'s and `unless`'s
/// right input, in any metric;
/// - a `Project` whose items reading an estimate ([`reads_an_estimate`])
/// each keep its error ([`keeps_error`]): $\beta = 1$, in any metric for
/// a copied column, in absolute error for `abs`, `clamp`, `clamp_min`,
/// `clamp_max` (also as the PromQL frontend's `promql_` functions) and a
/// negation, in relative error for a product with or quotient by a
/// nonzero literal (`q * 2`, `q / 1000`);
/// negation, in relative error for `abs` and a negation
/// ([`negation_or_abs`]) and a product with or quotient by a nonzero
/// literal (`q * 2`, `q / 1000`);
/// - a `Filter` whose predicate reads only columns copied from a source
/// (lineage `SourceColumn`, labels and keys): $\beta = 1$ in any metric;
/// - an `Aggregate` (or exact accumulator build) of counts ($\beta = 0$),
Expand Down Expand Up @@ -69,6 +75,13 @@ pub(crate) fn error_contract(
.filter(|item| reads_an_estimate(memo, *child, &item.expr))
.all(|item| keeps_error(&item.expr, metric))
.then(|| uniform(1.0)),
// `label_replace(v, "x", "$1", "job", "(.*)")` writes label `x`
// from label `job` and copies every value: a value's error is
// unchanged in any metric, while the label it writes reads no
// estimate.
NonASAPOp::PromqlRelabel { value, child, .. } => {
(!reads_an_estimate(memo, *child, value)).then(|| uniform(1.0))
}
NonASAPOp::Filter { pred, child } => (pred.0.operator_refs().is_empty()
&& copied_from_a_source(memo, *child, pred.0.columns_referenced()))
.then(|| uniform(1.0)),
Expand Down Expand Up @@ -110,6 +123,17 @@ pub(crate) fn error_contract(
}
Some(uniform(factor))
}
NonASAPOp::BinaryOp {
operator:
BinaryOperator {
kind: BinaryOpKind::Set(kind),
..
},
..
} => Some(ErrorContract {
error_factor_per_input: set_operator_error_factors(kind).to_vec(),
linear_bound_holds_up_to_per_input: vec![None; 2],
}),
NonASAPOp::BinaryOp {
operator:
BinaryOperator {
Expand Down Expand Up @@ -252,16 +276,19 @@ fn reads_an_estimate(memo: &Memo, child: MemoGroupId, expr: &ScalarExpr<MemoGrou
/// column it reads, with $\beta = 1$: a copy of a column or a literal, or
/// the series identity without its metric name (`promql_drop_metric_name`,
/// which changes a label, not a value), in any metric; a 1-Lipschitz
/// function of a column ([`one_lipschitz`]) in absolute error; a column
/// times or divided by a nonzero literal in relative error.
/// function of a column ([`one_lipschitz`]) in absolute error; a column's
/// negation or `abs` ([`negation_or_abs`]), or a column times or divided
/// by a nonzero literal, in relative error.
fn keeps_error<C>(expr: &ScalarExpr<C>, metric: AccuracyMetric) -> bool {
match expr {
ScalarExpr::Column(_) | ScalarExpr::Literal(_) => true,
ScalarExpr::FunctionCall { name, args } if name == "promql_drop_metric_name" => {
matches!(args[..], [ScalarExpr::Column(_)])
}
_ if matches!(metric, AccuracyMetric::Absolute { .. }) => one_lipschitz(expr),
_ if metric == AccuracyMetric::Relative => scaled_by_a_constant(expr),
_ if metric == AccuracyMetric::Relative => {
negation_or_abs(expr) || scaled_by_a_constant(expr)
}
_ => false,
}
}
Expand All @@ -271,19 +298,67 @@ fn keeps_error<C>(expr: &ScalarExpr<C>, metric: AccuracyMetric) -> bool {
/// `promql_abs`, …) of a column with literal bounds, or a column's
/// negation (1-Lipschitz).
fn one_lipschitz<C>(expr: &ScalarExpr<C>) -> bool {
negation_or_abs(expr)
|| function_of_a_column_and_literals(expr, &["clamp", "clamp_min", "clamp_max"])
}

/// Whether `expr` is a column's negation or `abs` (`promql_abs`). Each
/// moves the value by at most the column's error and keeps its magnitude,
/// so it keeps the error both absolutely and as a share of the value:
/// $|(-\hat{x}) - (-x)| = |\hat{x} - x|$ and
/// $||\hat{x}| - |x|| \le |\hat{x} - x|$ (the reverse triangle
/// inequality), while $|-x| = ||x|| = |x|$. With x = 10 and x̂ = 10.5 (5%
/// off), −x̂ = −10.5 is 5% off −10; with x = −10 and x̂ = −10.5, |x̂| =
/// 10.5 is 5% off 10.
fn negation_or_abs<C>(expr: &ScalarExpr<C>) -> bool {
match expr {
ScalarExpr::Negative { expr, .. } => matches!(**expr, ScalarExpr::Column(_)),
ScalarExpr::FunctionCall { name, args } => {
let name = name.to_ascii_lowercase();
matches!(
name.strip_prefix("promql_").unwrap_or(&name),
"abs" | "clamp" | "clamp_min" | "clamp_max"
) && matches!(args.first(), Some(ScalarExpr::Column(_)))
&& args[1..]
.iter()
.all(|arg| matches!(arg, ScalarExpr::Literal(_)))
}
_ => false,
_ => function_of_a_column_and_literals(expr, &["abs"]),
}
}

/// Whether `expr` calls one of `functions` (or the PromQL frontend's
/// `promql_` function of that name), any case, on a column and then
/// literals only: `promql_clamp(value, 0, 100)`.
fn function_of_a_column_and_literals<C>(expr: &ScalarExpr<C>, functions: &[&str]) -> bool {
let ScalarExpr::FunctionCall { name, args } = expr else {
return false;
};
const FRONTEND_PREFIX: &str = "promql_";
let name = match name.split_at_checked(FRONTEND_PREFIX.len()) {
Some((prefix, rest)) if prefix.eq_ignore_ascii_case(FRONTEND_PREFIX) => rest,
_ => name.as_str(),
};
functions
.iter()
.any(|function| function.eq_ignore_ascii_case(name))
&& matches!(args.first(), Some(ScalarExpr::Column(_)))
&& args[1..]
.iter()
.all(|arg| matches!(arg, ScalarExpr::Literal(_)))
}

/// $\beta$ of a PromQL set operator's left and right inputs. Each output
/// series is an input series, value copied, and which ones are kept
/// depends only on which labels the other side has: `lhs and rhs` and
/// `lhs unless rhs` keep left series, so the output's error is the left's
/// and the right's values are never read, $(1, 0)$; `lhs or rhs` copies
/// each value from one of them, $(1, 1)$. With q̂ = 10.5 for q = 10 on the
/// left, `q unless r` outputs 10.5 or nothing, whatever `r`'s values.
///
/// This assumes that which series each side has is exact: a row's presence
/// never depends on an estimate, since no operator with a contract drops
/// rows by value (a `Filter` on a computed value, `HAVING`, a ranking,
/// `count_values` and `absent` have none) and a readout emits the series
/// the exact evaluation does. So `q unless r` drops the series of `q` that
/// `r` has whether `r` is read from a sketch or exactly; were
/// `r = quantile_over_time(0.9, m[5m]) > 100` allowed, an estimate of 99
/// for 101 would drop the series from `r` and wrongly keep it in the
/// output.
fn set_operator_error_factors(kind: &PromQLVectorSetOpKind) -> [f64; 2] {
match kind {
PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless => [1.0, 0.0],
PromQLVectorSetOpKind::Or => [1.0, 1.0],
}
}

Expand Down
110 changes: 105 additions & 5 deletions crates/accuracy/src/error_contract/tests.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
use super::*;
use asap_types::ir::operator::arguments::Reduction;
use asap_types::ir::operator::arguments::{PromQLVectorSetOpKind, Reduction};
use asap_types::ir::scalar::CompareOpKind;
use asap_types::ir::schema::{DataType, PROMQL_SERIES_IDENTITY};
use asap_types::ir::{ExprSemantics, Predicate, ProjectItem};
use asap_types::stages::ValueSign;
use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate};
use asap_types::test_support::memo_fixtures::{aggregate, arithmetic, compare, readout, scan};
use asap_types::test_support::memo_fixtures::{
aggregate, arithmetic, compare, readout, scan, set_operator,
};
use asap_types::workload::{DataWorkload, SourceFacts};
use std::collections::BTreeMap;

Expand All @@ -15,8 +17,11 @@ use std::collections::BTreeMap;
/// `latency` by `job`, `job = "api"` reads such a column; `value > 0.5`
/// compares the computed estimate and has no contract. A `Project` of
/// `abs`, `clamp` or a negation of the value changes it by at most its
/// error: β = 1 for absolute errors only; one scaling it by a nonzero
/// constant keeps its relative error: β = 1 for relative errors. Only
/// error: β = 1 for absolute errors. `abs` and a negation also keep its
/// magnitude, so they keep its relative error too (β = 1): q = 10,
/// q̂ = 10.5 gives −q̂ = −10.5, 5% off −10. A clamp does not: it is not
/// judged in relative error. One scaling it by a nonzero constant keeps
/// its relative error: β = 1 for relative errors. Only
/// the items reading the estimate are judged, so the series identity
/// `promql_drop_metric_name` computes does not count.
#[test]
Expand Down Expand Up @@ -136,7 +141,17 @@ fn key_filters_and_value_preserving_projects_have_unit_contracts() {
Some(vec![1.0]),
"{expr:?}"
);
assert_eq!(factor(&project, AccuracyMetric::Relative), None, "{expr:?}");
let keeps_magnitude = match &expr {
ScalarExpr::Negative { .. } => true,
ScalarExpr::FunctionCall { name, .. } => name == "promql_abs",
_ => false,
};
assert_eq!(
factor(&project, AccuracyMetric::Relative),
keeps_magnitude.then(|| vec![1.0]),
"{expr:?}"
);
assert_eq!(factor(&project, AccuracyMetric::Rank), None, "{expr:?}");
}
// Scaling by a nonzero constant keeps a relative error: `q / 2`,
// `q * 2`, `2 * q`; scaling by 0 has no contract.
Expand Down Expand Up @@ -371,3 +386,88 @@ fn sums_and_averages_carry_relative_error_over_values_of_one_sign() {
);
}
}

/// PromQL operators that copy values and only choose which series to
/// keep, or rewrite a label, keep their input's error in every metric:
/// - `lhs and rhs` / `lhs unless rhs` keep or drop each left series by
/// whether a right series has its labels, and copy the kept value: the
/// output's error is the left's (β = 1), and the right's values are not
/// read (β = 0). With q̂ = 10.5 for q = 10 on the left, the output is
/// 10.5 or absent, whatever the right's values;
/// - `lhs or rhs` copies each value from one side: β = 1 on both;
/// - `label_replace(v, "x", "$1", "job", "(.*)")` writes label `x` from
/// label `job` and copies the value: β = 1. A rewrite reading the
/// estimate column itself has no contract.
#[test]
fn set_operators_and_relabels_copy_their_inputs_error() {
let mut memo = Memo::new(&DataWorkload::default());
let g1 = memo
.find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![]))
.unwrap();
let state = memo
.find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg {
child: g1,
summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch),
input: SummaryUpdate::column(ColumnRef::SampleValue),
reduction: Reduction::by(vec![2]),
filter: None,
}))
.unwrap();
let readout = memo
.find_or_add_group(readout(SketchStatistic::Quantile { q: 0.9 }, state))
.unwrap();
let lineage = &memo.groups()[readout.0 as usize]
.group_properties
.column_lineage;
let column_where = |copied: bool| {
lineage
.iter()
.position(|l| matches!(l, ColumnLineage::SourceColumn(_)) == copied)
.expect("the readout has a label and an estimate")
};
let (job, estimate) = (column_where(true), column_where(false));
let relabel = |source_column| {
Operator::NonASAP(NonASAPOp::PromqlRelabel {
dst: "x".into(),
value: ScalarExpr::FunctionCall {
name: "label_replace".into(),
args: vec![
ScalarExpr::Column(source_column),
ScalarExpr::Literal(ScalarValue::Utf8("(.*)".into())),
ScalarExpr::Literal(ScalarValue::Utf8("$1".into())),
],
},
child: readout,
})
};
let factors = |consumer: &Operator<MemoGroupId>, metric| {
error_contract(&memo, consumer, metric).map(|contract| contract.error_factor_per_input)
};
for metric in AccuracyMetric::ALL {
for kind in [PromQLVectorSetOpKind::And, PromQLVectorSetOpKind::Unless] {
assert_eq!(
factors(&set_operator(kind.clone(), readout, readout), metric),
Some(vec![1.0, 0.0]),
"{kind:?}, {metric:?}"
);
}
assert_eq!(
factors(
&set_operator(PromQLVectorSetOpKind::Or, readout, readout),
metric
),
Some(vec![1.0, 1.0]),
"or, {metric:?}"
);
assert_eq!(
factors(&relabel(job), metric),
Some(vec![1.0]),
"relabel from job, {metric:?}"
);
assert_eq!(
factors(&relabel(estimate), metric),
None,
"relabel from the estimate, {metric:?}"
);
}
}
Loading
Loading