Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/accuracy/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ edition = "2021"
[dependencies]
asap_sketchlib = { workspace = true }
asap-types = { path = "../types" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"

[dev-dependencies]
Expand Down
8 changes: 7 additions & 1 deletion crates/accuracy/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
//! theoretical guarantee of reading a statistic from a summary with given
//! parameters ([`theoretical_guarantee`], from the per-algorithm
//! guarantees in `guarantee`), the deployment's measured accuracy
//! ([`AccuracyEvidence`], in `evidence`), each operator's error contract
//! ([`AccuracyEvidence`], in `evidence`, and the measured table
//! [`MeasuredSketchAccuracy`], in `measured`), each operator's error contract
//! ([`ErrorContract`], in `error_contract`), the error metrics of a memo
//! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the
//! metric of a readout's error from `readout`), each readout's required
Expand All @@ -19,6 +20,7 @@ mod error_contract;
mod evidence;
mod guarantee;
mod logical_edge;
mod measured;
mod metric_rule;
mod readout;
mod readout_caps;
Expand All @@ -27,5 +29,9 @@ pub use error_contract::ErrorContract;
pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees};
pub use guarantee::theoretical_guarantee;
pub use logical_edge::within_limit;
pub use measured::{
ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy,
MeasuredSketchAccuracyError,
};
pub use metric_rule::ErrorMetricsOfMemo;
pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy};
97 changes: 97 additions & 0 deletions crates/accuracy/src/measured.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
//! The deployment's measured sketch accuracy: the errors a calibration run
//! measured on the executor's own sketches, which Pass 2 reads through
//! [`AccuracyEvidence`] before each summary's theoretical guarantee.
//!
//! Example: KLL with `k = 800`, read for `Quantile { q: 0.99 }`, measured
//! over 1,000 trials, has rank error 0.00176 at failure probability 0.01
//! and 0.00121 at 0.05. A request at failure probability 0.02 gets
//! 0.00176 (the largest recorded failure probability at most 0.02); a
//! request at 0.001 gets no measured guarantee, so the theoretical one
//! applies.

use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric};
use asap_types::ir::schema::FieldDataType;
use asap_types::summary_type::{SketchParams, SketchStatistic};
use serde::{Deserialize, Serialize};

use crate::AccuracyEvidence;

/// Errors measured on the executor's own sketches. Default: empty, every
/// sketch keeps its theoretical guarantee.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MeasuredSketchAccuracy {
pub entries: Vec<MeasuredAccuracyEntry>,
}

/// One sketch configuration, read for one statistic, in one metric.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MeasuredAccuracyEntry {
/// The exact parameters measured; only these match.
pub sketch: SketchParams,
/// The readout, e.g. `Quantile { q: 0.99 }`; only the same statistic
/// matches.
pub statistic: SketchStatistic,
/// The metric every recorded error is in, e.g. `Rank` for KLL.
pub accuracy_metric: AccuracyMetric,
/// Independent trials; every recorded failure probability is at least
/// 3 / trials.
pub trials: u32,
/// Ascending failure probability; the error does not increase as the
/// failure probability grows.
pub error_at: Vec<ErrorAtFailureProbability>,
}

/// One measured point: the error not exceeded except with probability
/// `failure_probability`.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ErrorAtFailureProbability {
pub failure_probability: f64,
/// The (1 − δ) quantile of the per-value errors, the worst over the
/// measured merge counts, in the entry's `accuracy_metric`.
pub error: f64,
}

/// Why a measured accuracy table is not valid.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub enum MeasuredSketchAccuracyError {
/// Entry `entry` has a failure probability outside (0, 1] or below
/// 3 / trials.
FailureProbabilityOutOfRange {
entry: usize,
failure_probability: f64,
},
/// Entry `entry` has a negative or non-finite error.
InvalidError { entry: usize, error: f64 },
/// Entry `entry`'s failure probabilities are not ascending, or its
/// error increases as the failure probability grows.
NotMonotone { entry: usize },
/// Entries `first` and `second` measure the same sketch, statistic and
/// metric.
DuplicateEntry { first: usize, second: usize },
}

impl MeasuredSketchAccuracy {
/// Check every entry: failure probabilities in (0, 1] and at least
/// 3 / trials, errors finite and non-negative, ascending failure
/// probability with non-increasing error, and no two entries for one
/// (sketch, statistic, metric).
pub fn validate(&self) -> Result<(), MeasuredSketchAccuracyError> {
todo!("asap_run: implement after test approval")
}
}

impl AccuracyEvidence for MeasuredSketchAccuracy {
fn measured_guarantee(
&self,
_summary_type: &FieldDataType,
_parameters: &SketchParams,
_statistic: &SketchStatistic,
_failure_probability: f64,
) -> Option<AccuracyGuarantee> {
todo!("asap_run: implement after test approval")
}
}
Loading
Loading