diff --git a/Cargo.lock b/Cargo.lock index 703605ad6..ff4a7412a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,35 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "aqpbm-core" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae#7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae" +dependencies = [ + "anyhow", + "aqpbm-datagen", + "bitflags", + "chrono", + "libc", + "rand 0.9.5", + "rand_xoshiro", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "aqpbm-datagen" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae#7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae" +dependencies = [ + "rand 0.9.5", + "rand_distr", + "rand_xoshiro", + "serde", + "thiserror 2.0.18", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -306,6 +335,8 @@ dependencies = [ name = "asap-devtools" version = "0.1.0" dependencies = [ + "aqpbm-core", + "aqpbm-datagen", "asap-cost", "asap-executor", "asap-frontend-metricsql", @@ -318,6 +349,7 @@ dependencies = [ "asap-stage2-physical-optimization", "asap-stage3-plan-selection", "asap-types", + "futures", "serde", "serde_json", "serde_yaml", @@ -2582,6 +2614,25 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.5", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "recursive" version = "0.1.1" diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 37c4ff86d..057e0bd9c 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -20,6 +20,13 @@ asap-planner = { path = "../planner" } # The executor's supported summary families, which `asap_planner` plans # with when the deployment declares none, as `plan_installable` does. asap-executor = { path = "../executor" } +# The engine-comparison harness of sketch-bench (`aqpbm_core::harness`; +# no default features: no sketch library, polars or CLI crate): `asap_run bench-external +# --lane asap` runs the Planner's executor as an engine under test +# (`asap_lane::AsapEngine`). Pinned to a sketch-bench commit; sketch-bench +# never depends on the planner. +aqpbm-core = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae", default-features = false } +futures = "0.3" # Used by the asap_planner / variant_coverage bins (catalog schemas, # async SQL path, JSON output) and by the topk_ir / canonical_examples @@ -31,6 +38,8 @@ serde_yaml = "0.9" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } [dev-dependencies] +# The data generator of the harness's workloads, at the harness's pin. +aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae" } asap-types = { path = "../types", features = ["test-support"] } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } @@ -39,6 +48,11 @@ asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } name = "devtools" path = "tests/main.rs" +# Its own process: the check compares thread and process CPU. +[[test]] +name = "asap_lane_conformance" +path = "tests/asap_lane_conformance.rs" + # The nightly corpus-snapshots workflow runs this binary by name. [[test]] name = "corpus_snapshots" diff --git a/crates/devtools/src/asap_lane/installed_plan.rs b/crates/devtools/src/asap_lane/installed_plan.rs new file mode 100644 index 000000000..75d8c8890 --- /dev/null +++ b/crates/devtools/src/asap_lane/installed_plan.rs @@ -0,0 +1,355 @@ +//! One installed plan of the ASAP lane: its compiled runs and its store. + +use std::sync::Arc; + +use aqpbm_core::harness::engine::{ + Answers, EngineError, Footprint, InstalledArm, PlanSummary, SampleChunk, StepReport, + WorkloadSpec, +}; +use asap_executor::physical_planner::CompiledPhysicalDAG; +use asap_executor::runtime::{Limits, RunContext, Scope}; +use asap_executor::sources::{DataStore, InMemoryDataStore}; +use asap_executor::values::Batch; +use asap_planner::{InstallablePlan, MaterializedView, MaterializedViewRefresh, RefreshWindow}; +use futures::executor::block_on; +use futures::StreamExt; + +use super::time_series_batches::{answers_of, time_series_tables, SampleRouter, TimeSeriesTable}; + +/// The memory one run may reserve: no limit in practice, so the lane +/// measures the plan, not a limit of this runtime. +const RUN_MEMORY_LIMIT_BYTES: usize = 1 << 40; + +/// An installed plan: the time series it reads, its ingestion refresh +/// groups and query run compiled once, and the store holding its raw +/// samples and stored views. +pub(super) struct InstalledPlan { + plan: InstallablePlan, + /// The workload query each root answers, in root order. + query_ids: Vec, + tables: Vec, + router: SampleRouter, + refreshes: Vec, + query_run: CompiledPhysicalDAG, + /// Shared with each run while it runs, held alone between steps. + store: Arc, + /// The end of the samples ingested so far: the data start before the + /// first chunk (the harness's samples lie in `(start, start + duration]`). + ingested_until_ms: i64, +} + +/// One ingestion refresh group: refreshed by one ingestion run per window +/// `(e - width, e]` with `e + phase ≡ 0 (mod width)`, whose roots are its +/// views. The phase is measured back from evaluation times on the width's +/// grid: the harness evaluates at multiples of the evaluation interval, +/// which plans made for evaluation phase 0 assume. +struct GroupRefresh { + window: RefreshWindow, + views: Vec, + compiled: CompiledPhysicalDAG, +} + +/// Bad input: the lane cannot run this plan, though the executor may. +fn unsupported(reason: impl std::fmt::Display) -> EngineError { + EngineError(format!("unsupported by the ASAP lane: {reason}")) +} + +/// The executor cannot compile this plan. +fn not_executable(reason: impl std::fmt::Display) -> EngineError { + EngineError(format!("not executable: {reason}")) +} + +fn run_failed(step: &str, reason: impl std::fmt::Display) -> EngineError { + EngineError(format!("{step}: {reason}")) +} + +impl InstalledPlan { + /// Compiles `plan`'s query run and each ingestion refresh group's + /// refresh, and binds an empty store: each time series the plan reads, + /// kept as far back as its query-time `Scan`s read and the refreshes + /// need. `query_ids` names the query of each root. + pub(super) fn install( + plan: InstallablePlan, + query_ids: Vec, + workload: &WorkloadSpec, + ) -> Result { + let query_run = plan.compile_query_run().map_err(not_executable)?; + let groups = plan.ingestion_refresh_groups().map_err(not_executable)?; + let mut refreshes = Vec::with_capacity(groups.len()); + for group in &groups { + let compiled = plan + .compile_ingestion_refresh(group) + .map_err(not_executable)?; + let first = group.views[0].node_id; + let window = group + .refresh_window + .ok_or_else(|| unsupported(format!("view {first} has no bounded window")))?; + refreshes.push(GroupRefresh { + window, + views: group.views.clone(), + compiled, + }); + } + let views = plan.materialized_views().map_err(not_executable)?; + if let Some(view) = views + .iter() + .find(|view| view.refresh() == MaterializedViewRefresh::OnFirstRead) + { + return Err(unsupported(format!( + "view {} is refreshed on first read", + view.node_id + ))); + } + let tables = time_series_tables(&plan.dag).map_err(unsupported)?; + let retention = plan.raw_data_retention().map_err(not_executable)?; + // A refresh reads the window ending at a chunk's end or before it, + // so the raw samples it reads reach back one window and one chunk; + // the harness's chunks are one evaluation interval each + // (`ArmSchedule::for_workload`). + let chunk_ms = workload + .queries + .iter() + .map(|query| query.evaluation_interval_ms) + .max() + .unwrap_or(0); + let refresh_lookback_ms = refreshes + .iter() + .map(|refresh| refresh.window.width_ms + chunk_ms) + .max() + .unwrap_or(0); + let mut store = InMemoryDataStore::default(); + for table in &tables { + let query_lookback_ms = retention + .iter() + .filter(|kept| kept.source() == &table.source) + .map(|kept| kept.lookback_ms()) + .try_fold(0, |longest, lookback| Some(longest.max(lookback?))); + let lookback_ms = query_lookback_ms.map(|ms| ms.max(refresh_lookback_ms)); + store = store + .with_appended_raw_data(table.source.clone(), table.schema.clone(), lookback_ms) + .map_err(unsupported)?; + } + let ingested_until_ms = workload + .data + .iter() + .map(|record| record.description.start_ms) + .min() + .unwrap_or(0); + let mut store = Arc::new(store); + // The windows before the data that a view keeps, refreshed empty, + // so a query shifted back past the data start reads windows with + // no samples rather than missing ones. Example: a 10 s chain kept + // 5 min, with data from 0, has empty windows ending at -290 s ... + // 0 s before the first chunk. + for refresh in &refreshes { + let kept_ms = refresh + .views + .iter() + .map(MaterializedView::retention_ms) + .max(); + let from_ms = ingested_until_ms - kept_ms.unwrap_or(0) as i64; + for end_ms in window_ends(refresh.window, from_ms, ingested_until_ms) { + refresh_window(refresh, &mut store, end_ms)?; + } + } + Ok(InstalledPlan { + plan, + query_ids, + tables, + router: SampleRouter::default(), + refreshes, + query_run, + store, + ingested_until_ms, + }) + } +} + +/// The store, between runs: no run holds it then. +fn store_between_runs(store: &mut Arc) -> &mut InMemoryDataStore { + Arc::get_mut(store).expect("no run holds the store between steps") +} + +/// Refreshes `refresh` for the window ending at `end_ms`: one ingestion run +/// whose roots are its views, each view's output appended to `store`. +fn refresh_window( + refresh: &GroupRefresh, + store: &mut Arc, + end_ms: i64, +) -> Result<(), EngineError> { + let start_ms = end_ms - refresh.window.width_ms as i64; + let scope = Scope::Ingestion { + window_start_ms: start_ms, + window_end_ms: end_ms, + }; + let outputs = run_ingestion(&refresh.compiled, store, scope) + .map_err(|error| run_failed(&format!("refresh of ({start_ms}, {end_ms}]"), error))?; + let store = store_between_runs(store); + for (view, batches) in refresh.views.iter().zip(outputs) { + store + .append_materialized_view(view, (start_ms, end_ms), batches) + .map_err(|error| run_failed("storing a view", error))?; + } + Ok(()) +} + +/// `compiled` run in ingestion `scope` over `store`: each root's batches. +fn run_ingestion( + compiled: &CompiledPhysicalDAG, + store: &Arc, + scope: Scope, +) -> Result>, asap_executor::Error> { + let instance = compiled.instantiate(Arc::clone(store) as Arc)?; + let context = RunContext::new(scope, run_limits())?; + let outputs = block_on(instance.execute_ingestion(compiled.roots(), context)?)?; + Ok(outputs + .into_iter() + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .collect()) +} + +/// `compiled` run at `evaluation_time_ms` over `store`: each root's batches. +fn run_query( + compiled: &CompiledPhysicalDAG, + store: &Arc, + evaluation_time_ms: i64, +) -> Result>, asap_executor::Error> { + let instance = compiled.instantiate(Arc::clone(store) as Arc)?; + let context = RunContext::new(Scope::Query { evaluation_time_ms }, run_limits())?; + let outputs = instance.execute(compiled.roots(), context)?; + block_on(async { + let mut per_root = Vec::with_capacity(outputs.len()); + for mut output in outputs { + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + batches.push((*batch?).clone()); + } + per_root.push(batches); + } + Ok(per_root) + }) +} + +fn run_limits() -> Limits { + Limits { + max_bytes: RUN_MEMORY_LIMIT_BYTES, + } +} + +/// The ends `e` in `(after_ms, until_ms]` of `window`'s windows: `e + +/// phase ≡ 0 (mod width)`, since a window ending `phase` before an +/// evaluation on the width's grid ends there. Example: width 10 s, phase 0, +/// `(0, 30 s]`: 10, 20 and 30 s. +fn window_ends(window: RefreshWindow, after_ms: i64, until_ms: i64) -> impl Iterator { + let width = window.width_ms as i64; + let phase = window.phase_ms as i64; + let first = after_ms + (-phase - after_ms).rem_euclid(width); + let first = if first == after_ms { + first + width + } else { + first + }; + (0..) + .map(move |k| first + k * width) + .take_while(move |&end| end <= until_ms) +} + +impl InstalledArm for InstalledPlan { + /// One line per DAG node (id, operator, timing, lifetime, retention), + /// its operators, and its stored views; the lane runs the plan as + /// given, so no substitutions and no sharing lost. + fn plan_summary(&self) -> PlanSummary { + let dag = &self.plan.dag; + let text = dag + .nodes + .iter() + .map(|node| { + format!( + "n{} {} {:?} {:?} retention_ms {:?}", + node.id, + node.payload.kind_name(), + node.output_state.timing, + node.lifetime, + node.retention_ms + ) + }) + .chain(std::iter::once(format!("roots {:?}", dag.roots))) + .collect::>() + .join("\n"); + let stored_outputs = self + .plan + .materialized_views() + .unwrap_or_default() + .iter() + .map(|view| { + format!( + "n{} refreshed {:?}, kept {} ms", + view.node_id, + view.refresh(), + view.retention_ms() + ) + }) + .collect(); + PlanSummary { + text, + node_kinds: dag + .nodes + .iter() + .map(|node| node.payload.kind_name().to_string()) + .collect(), + stored_outputs, + substitutions: Vec::new(), + sharing_lost: Vec::new(), + } + } + + /// Appends the chunk's samples of each time series the plan reads as + /// its rows for `(previous end, chunk end]` (nothing for a series with + /// no sample in it), then refreshes each group for every window ending + /// in that range. + fn ingest(&mut self, chunk: &SampleChunk) -> Result { + let (after_ms, until_ms) = (self.ingested_until_ms, chunk.end_ms); + let routed = self.router.route(chunk, &mut self.tables); + for (table, samples) in self.tables.iter_mut().zip(routed) { + if samples.is_empty() { + continue; + } + let batch = table + .batch(&samples) + .map_err(|error| run_failed("raw samples", error))?; + store_between_runs(&mut self.store) + .append_raw_data(&table.source, (after_ms, until_ms), vec![batch]) + .map_err(|error| run_failed("raw samples", error))?; + } + for refresh in &self.refreshes { + for end_ms in window_ends(refresh.window, after_ms, until_ms) { + refresh_window(refresh, &mut self.store, end_ms)?; + } + } + self.ingested_until_ms = until_ms; + Ok(StepReport::default()) + } + + /// One query run at `at_ms`: root `i`'s rows answer query `i`. + fn evaluate(&mut self, at_ms: i64) -> Result<(Answers, StepReport), EngineError> { + let per_root = run_query(&self.query_run, &self.store, at_ms) + .map_err(|error| run_failed(&format!("query run at {at_ms}"), error))?; + let mut answers = Answers::default(); + for (query_id, batches) in self.query_ids.iter().zip(&per_root) { + let series = answers_of(batches).map_err(|error| run_failed(query_id, error))?; + answers.0.insert(query_id.clone(), series); + } + Ok((answers, StepReport::default())) + } + + /// The bytes of the stored views and of the raw samples held, as the + /// executor counts each batch: a dictionary of series texts shared by + /// several batches is counted in each. + fn footprint(&self) -> Result { + Ok(Footprint { + view_bytes: self.store.materialized_view_bytes() as u64, + raw_retained_bytes: self.store.raw_data_bytes() as u64, + peak_bytes: None, + }) + } +} diff --git a/crates/devtools/src/asap_lane/mod.rs b/crates/devtools/src/asap_lane/mod.rs new file mode 100644 index 000000000..9c988c867 --- /dev/null +++ b/crates/devtools/src/asap_lane/mod.rs @@ -0,0 +1,135 @@ +//! The in-process ASAP lane of the engine comparison: the Planner's +//! executor as an engine under test of the sketch-bench harness +//! (`aqpbm_core::harness`). +//! +//! [`AsapEngine`] installs a plan file `asap_planner --plan-file` wrote (an +//! [`InstallablePlan`]) as given, so this lane and every other lane of a +//! run execute the same DAG; `asap_run bench-external --lane asap` runs it +//! over a run directory. The lane is a deployment runtime over the +//! executor's public API: it keeps the raw samples and the stored views in +//! one [`InMemoryDataStore`](asap_executor::sources::InMemoryDataStore), refreshes each ingestion refresh group in one +//! ingestion run per window as samples arrive, and answers each evaluation +//! with one query run, all on the calling thread. +//! +//! Example: a plan for `sum_over_time(m[5m])` every 10 s built at ingestion +//! time is a chain of 10 s tumbling windows: each ingested chunk of 10 s +//! refreshes the window ending at its end, and the query run at `T` merges +//! the 30 windows ending in `(T - 5 min, T]`. + +mod installed_plan; +mod time_series_batches; + +use std::path::PathBuf; + +use aqpbm_core::harness::engine::{ + ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, + WorkloadSpec, +}; +use asap_planner::InstallablePlan; + +use installed_plan::InstalledPlan; + +/// The Planner's executor as an engine under test: installs the plan file +/// named by `arm.engine_options["plan_file"]`. +/// +/// Its version is the commit in `ASAP_PLANNER_COMMIT` at build time +/// (`unknown` without it), dirty when `ASAP_PLANNER_DIRTY` is `1`; e.g. +/// `ASAP_PLANNER_COMMIT=$(git rev-parse HEAD) cargo build --release --bin asap_run`. +#[derive(Debug, Clone, Copy, Default)] +pub struct AsapEngine; + +impl EngineUnderTest for AsapEngine { + fn description(&self) -> EngineDescription { + EngineDescription { + id: "asap".into(), + version: option_env!("ASAP_PLANNER_COMMIT") + .unwrap_or("unknown") + .into(), + version_dirty: option_env!("ASAP_PLANNER_DIRTY") == Some("1"), + build_profile: if cfg!(debug_assertions) { + "debug" + } else { + "release" + } + .into(), + } + } + + /// Every ingestion and query run runs on the thread that calls + /// `ingest` or `evaluate` (`futures::executor::block_on`). + fn cpu_accounting(&self) -> CpuAccounting { + CpuAccounting::CallingThread + } + + /// Reads and validates the plan file, matches its query texts to + /// `workload`'s queries, and compiles it with the executor. A plan the + /// executor cannot compile is an error starting `not executable:`, so + /// the cell reads "not run: not executable: ..." while the other lanes + /// still run that plan. + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let plan_file = plan_file_of(arm)?; + let plan = read_plan_file(&plan_file)?; + let query_ids = query_ids_of(&plan, workload) + .map_err(|error| EngineError(format!("plan file {}: {error}", plan_file.display())))?; + Ok(Box::new(InstalledPlan::install(plan, query_ids, workload)?)) + } +} + +/// `arm.engine_options["plan_file"]`. +fn plan_file_of(arm: &ArmSpec) -> Result { + arm.engine_options + .get("plan_file") + .and_then(serde_json::Value::as_str) + .map(PathBuf::from) + .ok_or_else(|| { + EngineError(format!( + "arm {}: engine options {} name no plan_file", + arm.name, arm.engine_options + )) + }) +} + +/// The plan file at `path`, read and validated. +fn read_plan_file(path: &std::path::Path) -> Result { + let error = |error: &dyn std::fmt::Display| { + EngineError(format!("plan file {}: {error}", path.display())) + }; + let text = std::fs::read_to_string(path).map_err(|e| error(&e))?; + let plan = InstallablePlan::from_json(&text).map_err(|e| error(&e))?; + plan.validate().map_err(|e| error(&e))?; + Ok(plan) +} + +/// The id of the query each root of `plan` answers: root `i` answers query +/// text `i`, which must be one of the texts (in any language) of +/// `workload`'s query `i`. Example: a plan for `sum_over_time(m[5m])` and a +/// workload whose query `sum` reads `sum_over_time(m[5m])` give `["sum"]`. +fn query_ids_of(plan: &InstallablePlan, workload: &WorkloadSpec) -> Result, String> { + if plan.query_texts.len() != workload.queries.len() { + return Err(format!( + "{} query text(s) for workload {}'s {} queries", + plan.query_texts.len(), + workload.name, + workload.queries.len() + )); + } + plan.query_texts + .iter() + .zip(&workload.queries) + .enumerate() + .map(|(index, (planned, query))| { + if query.texts.iter().any(|text| &text.text == planned) { + return Ok(query.id.clone()); + } + let texts: Vec<&str> = query.texts.iter().map(|text| text.text.as_str()).collect(); + Err(format!( + "query {index} ({}) is planned as {planned:?}, but the workload's texts are {texts:?}", + query.id + )) + }) + .collect() +} diff --git a/crates/devtools/src/asap_lane/time_series_batches.rs b/crates/devtools/src/asap_lane/time_series_batches.rs new file mode 100644 index 000000000..3ca5e72e2 --- /dev/null +++ b/crates/devtools/src/asap_lane/time_series_batches.rs @@ -0,0 +1,244 @@ +//! Samples in, answers out: the raw table of each time series the plan's +//! `Scan`s read, built column by column from the harness's samples, and the +//! series and value of each answer row. +//! +//! A time series is the table `(ts, value, $promql_series_identity, +//! *labels)` over its samples (the columns its `Scan`s read): the series +//! identity is the canonical JSON of every label, `__name__` included, +//! e.g. `{"__name__":"m","s":"s_0"}`. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use aqpbm_core::harness::data::{Sample, SampleChunk, SeriesLabels}; +use asap_executor::arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array, +}; +use asap_executor::arrow::datatypes::UInt32Type; +use asap_executor::values::{Batch, BatchColumn, SchemaRef, Value}; +use asap_planner::PhysicalASAPDAG; +use asap_types::ir::operator::Source; +use asap_types::ir::schema::{DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; + +/// One time series' raw table: what each of its columns holds, and the +/// series seen so far, each with an ordinal that is its dictionary key in +/// every text column. +pub(super) struct TimeSeriesTable { + /// `Source::TimeSeries` of its metric. + pub(super) source: Source, + /// The union of its `Scan`s' columns. + pub(super) schema: SchemaRef, + columns: Vec, + /// The series seen so far, by ordinal. + series: Vec>, + /// Each column's dictionary texts, one per series of `series` (`None` + /// for a column that is not text); rebuilt when a series is added. + texts: Option>>, +} + +/// What one column of a time series' raw table holds. +enum TimeSeriesColumn { + /// The sample's time, ms. + Time, + /// The sample's value. + Value, + /// The canonical JSON of the series' labels. + SeriesIdentity, + /// The value of this label; empty when the series lacks it. + Label(String), +} + +/// The raw table of each time series `dag`'s `Scan`s read, in the order of +/// its first `Scan` ([`PhysicalASAPDAG::raw_data_schemas`]). `Err` for a +/// source the lane cannot feed: a SQL table, or a column that is neither +/// the time, the value, the series identity nor a text label. +pub(super) fn time_series_tables(dag: &PhysicalASAPDAG) -> Result, String> { + dag.raw_data_schemas() + .into_iter() + .map(|(source, schema)| { + if !matches!(source, Source::TimeSeries { .. }) { + return Err(format!("the plan scans {source:?}, not a time series")); + } + let columns = schema + .fields + .iter() + .enumerate() + .map(|(index, field)| match (&field.dtype, field.name.as_str()) { + _ if Some(index) == schema.time_index => Ok(TimeSeriesColumn::Time), + (FieldDataType::Plain(DataType::Float64), "value") => { + Ok(TimeSeriesColumn::Value) + } + (FieldDataType::Plain(DataType::Utf8), PROMQL_SERIES_IDENTITY) => { + Ok(TimeSeriesColumn::SeriesIdentity) + } + (FieldDataType::Plain(DataType::Utf8), label) => { + Ok(TimeSeriesColumn::Label(label.to_string())) + } + (dtype, name) => Err(format!( + "{source:?}: column {name} of type {dtype:?} is not a label" + )), + }) + .collect::>()?; + Ok(TimeSeriesTable { + source, + schema: Arc::new(schema), + columns, + series: Vec::new(), + texts: None, + }) + }) + .collect() +} + +/// Where each series' samples go: its table and its ordinal there, or +/// nowhere (a metric the plan does not read), looked up once per series. +/// The harness shares one label map among a series' samples, so the map's +/// address names the series; the map is kept alive here, so its address is +/// never reused by another series. +#[derive(Default)] +pub(super) struct SampleRouter { + by_address: HashMap<*const SeriesLabels, Option<(usize, u32)>>, + /// Every series seen, keeping its address taken. + seen: Vec>, +} + +impl SampleRouter { + /// `chunk`'s samples per table of `tables`, each with its series' + /// ordinal, in chunk order; a sample of a metric no table reads is + /// dropped. + pub(super) fn route<'c>( + &mut self, + chunk: &'c SampleChunk, + tables: &mut [TimeSeriesTable], + ) -> Vec> { + let mut routed = vec![Vec::new(); tables.len()]; + for sample in &chunk.samples { + let route = *self + .by_address + .entry(Arc::as_ptr(&sample.labels)) + .or_insert_with(|| { + self.seen.push(Arc::clone(&sample.labels)); + let metric = sample.labels.get("__name__")?; + let table = tables.iter().position(|table| { + matches!(&table.source, Source::TimeSeries { metric: read } if read == metric) + })?; + Some((table, tables[table].add_series(&sample.labels))) + }); + if let Some((table, ordinal)) = route { + routed[table].push((ordinal, sample)); + } + } + routed + } +} + +impl TimeSeriesTable { + /// Adds a series; its ordinal. + fn add_series(&mut self, labels: &Arc) -> u32 { + self.series.push(Arc::clone(labels)); + self.texts = None; + (self.series.len() - 1) as u32 + } + + /// Each text column's dictionary texts, one per series, built when a + /// series was added since the last call. + fn texts(&mut self) -> &[Option] { + let series = &self.series; + self.texts.get_or_insert_with(|| { + self.columns + .iter() + .map(|column| { + let texts: StringArray = match column { + TimeSeriesColumn::Time | TimeSeriesColumn::Value => return None, + TimeSeriesColumn::SeriesIdentity => series + .iter() + .map(|labels| { + Some( + serde_json::to_string(labels.as_ref()) + .expect("labels are JSON"), + ) + }) + .collect(), + TimeSeriesColumn::Label(label) => series + .iter() + .map(|labels| Some(labels.get(label).map_or("", String::as_str))) + .collect(), + }; + Some(Arc::new(texts) as ArrayRef) + }) + .collect() + }) + } + + /// The rows of `samples` (this table's, each with its series' ordinal) + /// in the table's schema. Every text column is dictionary-encoded: the + /// series' ordinal is the key, and the dictionary holds every series + /// seen so far, shared by every batch until a series is added. + pub(super) fn batch( + &mut self, + samples: &[(u32, &Sample)], + ) -> Result { + let keys = UInt32Array::from_iter_values(samples.iter().map(|(ordinal, _)| *ordinal)); + let schema = self.schema.clone(); + let mut columns = Vec::with_capacity(self.columns.len()); + // The texts are `Arc`s: cloning them shares the dictionary. + let texts = self.texts().to_vec(); + for (column, texts) in self.columns.iter().zip(texts) { + columns.push(match (column, texts) { + (TimeSeriesColumn::Time, _) => { + BatchColumn::Plain(Arc::new(TimestampMillisecondArray::from_iter_values( + samples.iter().map(|(_, sample)| sample.ts_ms), + ))) + } + (TimeSeriesColumn::Value, _) => BatchColumn::Plain(Arc::new( + Float64Array::from_iter_values(samples.iter().map(|(_, sample)| sample.value)), + )), + (_, texts) => { + let texts = texts.expect("a text column has its texts"); + let column = DictionaryArray::::try_new(keys.clone(), texts) + .map_err(|error| asap_executor::Error::Invalid(error.to_string()))?; + BatchColumn::Plain(Arc::new(column)) + } + }); + } + Batch::try_from_columns(schema, columns) + } +} + +/// Each answer row of `batches` (one query root's output) as its series +/// labels and value: the series identity's labels, when the output keeps +/// one, and every other non-empty text column (a label the query grouped +/// by); the value is the row's first Float64 column. Example: a row +/// `(ts, 12.5, {"s":"s_0"})` of `sum_over_time(m[5m])` is `{s="s_0"} 12.5`. +/// `Err` for two rows of one series, which a query answer never has. +pub(super) fn answers_of(batches: &[Batch]) -> Result, String> { + let mut answers = BTreeMap::new(); + for batch in batches { + let schema = batch.schema(); + for row in batch.to_rows() { + let mut labels = SeriesLabels::new(); + let mut value = None; + for (field, cell) in schema.fields.iter().zip(&row) { + match cell { + Value::Utf8(text) if field.name == PROMQL_SERIES_IDENTITY => { + let identity: SeriesLabels = serde_json::from_str(text) + .map_err(|error| format!("series identity {text}: {error}"))?; + labels.extend(identity); + } + Value::Utf8(text) if !text.is_empty() => { + labels.insert(field.name.clone(), text.to_string()); + } + Value::Float64(number) if value.is_none() => value = Some(*number), + _ => {} + } + } + let value = value.ok_or_else(|| format!("an answer row without a value: {row:?}"))?; + if let Some(earlier) = answers.insert(labels.clone(), value) { + return Err(format!( + "two answers for series {labels:?}: {earlier} and {value}" + )); + } + } + } + Ok(answers) +} diff --git a/crates/devtools/src/bin/asap_run.rs b/crates/devtools/src/bin/asap_run.rs new file mode 100644 index 000000000..40e633cb9 --- /dev/null +++ b/crates/devtools/src/bin/asap_run.rs @@ -0,0 +1,124 @@ +// asap_run: runs the Planner's executor as one lane of the engine comparison +// of sketch-bench's harness (`aqpbm_core::harness`). +// +// asap_run bench-external --lane asap --run-dir [--variant ]... +// [--core ] [--warmup ] [--runs ] [--max-core-busy ] +// +// Runs every (workload, variant) cell of the run directory another step +// prepared (`workloads/.json`, `plans//.plan.json` written by +// `asap_planner --plan-file`) on `asap_lane::AsapEngine`, which installs each +// plan file as given, and writes `results/asap/__.cell.json` and the +// arm results beside them, as `approxbench bench-external run` does for +// its lanes; `approxbench bench-external report` then joins every lane. +// - `--variant`: only these variants (repeatable); every one by default. +// - `--core`: the core each step is pinned to (default 0). +// - `--warmup`, `--runs`: runs discarded, and measured (odd), per step +// (the harness's defaults). +// - `--max-core-busy`: a core busier than this fraction before a step +// refuses it (the harness's default). +// Prints one line per cell. Exits 2 on a usage error, 1 when the lane +// cannot run (a cell that fails is written as not run, and the next runs). + +use std::path::PathBuf; +use std::process::ExitCode; + +use aqpbm_core::harness::bench_external::{run_lane, LaneSetup}; +use aqpbm_core::harness::engine::EngineUnderTest; +use aqpbm_core::harness::protocol::{ + load_probe_for, MeasurementClocks, OsCpuClock, ProcessCpuFiles, Protocol, +}; +use aqpbm_core::harness::rundir::RunDir; +use asap_devtools::asap_lane::AsapEngine; + +const USAGE: &str = + "usage: asap_run bench-external --lane asap --run-dir [--variant ]... \ +[--core ] [--warmup ] [--runs ] [--max-core-busy ]"; + +enum Failure { + Usage(String), + Run(String), +} + +fn main() -> ExitCode { + match run(std::env::args().skip(1).collect()) { + Ok(()) => ExitCode::SUCCESS, + Err(Failure::Usage(error)) => { + eprintln!("asap_run: {error}\n{USAGE}"); + ExitCode::from(2) + } + Err(Failure::Run(error)) => { + eprintln!("asap_run: {error}"); + ExitCode::FAILURE + } + } +} + +fn run(args: Vec) -> Result<(), Failure> { + let mut args = args.into_iter(); + match args.next().as_deref() { + Some("bench-external") => {} + Some(other) => return Err(Failure::Usage(format!("unknown command {other}"))), + None => return Err(Failure::Usage("a command is required".into())), + } + let mut lane = None; + let mut run_dir = None; + let mut variants = Vec::new(); + let mut core = 0; + let mut protocol = Protocol::default(); + while let Some(flag) = args.next() { + let value = args + .next() + .ok_or_else(|| Failure::Usage(format!("{flag} needs a value")))?; + let number = |value: &str| -> Result { + value + .parse() + .map_err(|e| Failure::Usage(format!("{flag} {value}: {e}"))) + }; + match flag.as_str() { + "--lane" => lane = Some(value), + "--run-dir" => run_dir = Some(PathBuf::from(value)), + "--variant" => variants.push(value), + "--core" => core = number(&value)? as usize, + "--warmup" => protocol.warmup_runs = number(&value)?, + "--runs" => protocol.measured_runs = number(&value)?, + "--max-core-busy" => { + protocol.max_core_busy_fraction = value + .parse() + .map_err(|e| Failure::Usage(format!("{flag} {value}: {e}")))? + } + other => return Err(Failure::Usage(format!("unknown argument {other}"))), + } + } + let engine = match lane.as_deref() { + Some("asap") => AsapEngine, + Some(other) => { + return Err(Failure::Usage(format!( + "unknown lane {other}; known lanes: asap" + ))) + } + None => return Err(Failure::Usage("--lane is required".into())), + }; + let run_dir = run_dir.ok_or_else(|| Failure::Usage("--run-dir is required".into()))?; + let run_dir = RunDir::open(&run_dir).map_err(|e| Failure::Run(e.to_string()))?; + let clocks = MeasurementClocks::with_calling_thread(&OsCpuClock); + let mut probe = load_probe_for(&engine.cpu_accounting(), &ProcessCpuFiles::os()) + .map_err(|e| Failure::Run(e.to_string()))?; + let only = (!variants.is_empty()).then_some(variants.as_slice()); + let cells = run_lane( + &run_dir, + &engine, + only, + LaneSetup { + protocol: &protocol, + core, + probe: probe.as_mut(), + clocks: &clocks, + args: std::env::args().collect(), + }, + ) + .map_err(|e| Failure::Run(e.to_string()))?; + for cell in cells { + println!("{} {} {:?}", cell.workload, cell.variant, cell.outcome); + } + Ok(()) +} diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index de505a8cf..d2e1ae4c5 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -7,8 +7,11 @@ //! query corpora and holds their SQL catalogs. [`rqe_atomic_costs`] converts //! sketch-bench's measured per-operation costs into plan selection's //! measured table. [`planner_fuzz`] is the metamorphic fuzz tester for -//! missed reuse and sharing behind the `planner_fuzz` bin. +//! missed reuse and sharing behind the `planner_fuzz` bin. [`asap_lane`] is +//! the in-process ASAP lane of the engine comparison behind the `asap_run` +//! bin. +pub mod asap_lane; pub mod candidate_labels; pub mod corpora; pub mod planner_fuzz; diff --git a/crates/devtools/tests/asap_lane.rs b/crates/devtools/tests/asap_lane.rs new file mode 100644 index 000000000..c217218fa --- /dev/null +++ b/crates/devtools/tests/asap_lane.rs @@ -0,0 +1,407 @@ +//! The in-process ASAP lane of the engine comparison: `AsapEngine` installs a +//! plan file `asap_planner --plan-file` wrote and runs it with the Planner's +//! executor under the harness of sketch-bench (`aqpbm_core::harness`), as one +//! lane of `asap_run bench-external`. +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use aqpbm_core::harness::arm_run::ArmSchedule; +use aqpbm_core::harness::bench_external::{run_lane, CellOutcome, LaneSetup}; +use aqpbm_core::harness::data::{workload_chunks, DataRecord, SeriesLabels}; +use aqpbm_core::harness::engine::{ + ArmSpec, EngineError, EngineUnderTest, QueryLanguage, QueryText, WorkloadQuery, WorkloadSpec, +}; +use aqpbm_core::harness::protocol::{ + LoadProbe, MeasurementClocks, OsCpuClock, Protocol, ProtocolError, +}; +use aqpbm_core::harness::records::ArmResult; +use aqpbm_core::harness::rundir::RunDir; +use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, UniformParameter}; +use asap_devtools::asap_lane::AsapEngine; + +#[path = "asap_lane_support.rs"] +mod support; +use support::{planning_workload, TempDir}; + +/// Every core idle, so a test never waits on the machine's load. +struct IdleProbe; + +impl LoadProbe for IdleProbe { + fn core_busy_fraction(&mut self, _core: usize) -> Result { + Ok(0.0) + } + fn loadavg(&mut self) -> f64 { + 0.0 + } +} + +/// Writes `plan_file` with `asap_planner --plan-file` for `planning`, which +/// must select a plan; returns the planner's standard error, which notes +/// when the Planner's executor cannot run the plan. +fn write_plan_file(planning: &str, plan_file: &Path) -> String { + let workload_file = plan_file.with_extension("planning.json"); + std::fs::write(&workload_file, planning).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_asap_planner")) + .args(["--workload", workload_file.to_str().unwrap()]) + .args(["--run-until", "stage3", "--format", "json"]) + .args(["--plan-file", plan_file.to_str().unwrap()]) + .output() + .unwrap(); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(output.status.success(), "{stderr}"); + assert!(plan_file.exists(), "{stderr}"); + stderr +} + +/// The query of workload `w`: a 5-minute window, so a plan built at +/// ingestion time reads 30 tumbling windows of 10 s. +const SUM_QUERY: &str = "sum_over_time(m[5m])"; + +/// Workload `name` of the harness: query `sum` = `query` (PromQL) every 10 s +/// over `window_ms`, over `m` with 2 series (label `s`), one sample per +/// second from 0 for 360 s, values uniform in [0, 10), seed 1. +fn harness_workload(name: &str, query: &str, window_ms: u64) -> WorkloadSpec { + WorkloadSpec { + name: name.into(), + queries: vec![WorkloadQuery { + id: "sum".into(), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: query.into(), + }], + window_ms, + evaluation_interval_ms: 10_000, + }], + data: vec![DataRecord { + metric: "m".into(), + description: TimeSeriesDescription { + labels: vec![SeriesLabelSpec { + name: "s".into(), + cardinality: 2, + skewness: 0.0, + }], + series_num: 2, + start_ms: 0, + sample_interval_ms: 1_000, + duration_ms: 360_000, + value_distribution: DataDistribution::Uniform(UniformParameter { + lower_bound: 0.0, + upper_bound: 10.0, + seed: 0, + }), + counter: false, + seed: 1, + }, + }], + } +} + +fn write_json(path: &Path, value: &impl serde::Serialize) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, serde_json::to_string_pretty(value).unwrap()).unwrap(); +} + +/// A run directory with `workloads` and, per variant (substitutions not +/// allowed), the plan file of each workload in it. +fn run_dir(temp: &TempDir, workloads: &[&WorkloadSpec], plans: &[(&str, &str, PathBuf)]) -> RunDir { + let run = RunDir::create_at(&temp.0.join("run")).unwrap(); + for workload in workloads { + write_json( + &run.path() + .join("workloads") + .join(format!("{}.json", workload.name)), + workload, + ); + } + for (variant, workload, plan_file) in plans { + let dir = run.path().join("plans").join(variant); + write_json( + &dir.join("variant.json"), + &serde_json::json!({"substitutions_allowed": false}), + ); + std::fs::copy(plan_file, dir.join(format!("{workload}.plan.json"))).unwrap(); + } + run +} + +/// Runs the ASAP lane over `run` with warm-up `warmup` and `runs` measured +/// runs. +fn run_asap_lane( + run: &RunDir, + warmup: u32, + runs: u32, +) -> Vec { + let protocol = Protocol { + warmup_runs: warmup, + measured_runs: runs, + max_core_busy_fraction: 1.0, + }; + let clocks = MeasurementClocks::with_calling_thread(&OsCpuClock); + run_lane( + run, + &AsapEngine, + None, + LaneSetup { + protocol: &protocol, + core: allowed_core(), + probe: &mut IdleProbe, + clocks: &clocks, + args: vec!["test".into()], + }, + ) + .unwrap() +} + +/// A core this process may run on: the first of its `Cpus_allowed_list`. +fn allowed_core() -> usize { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + let list = status + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim(); + list.split([',', '-']).next().unwrap().parse().unwrap() +} + +/// `labels` without `__name__`: a PromQL `*_over_time` drops the metric name. +fn without_name(labels: &SeriesLabels) -> SeriesLabels { + let mut labels = labels.clone(); + labels.remove("__name__"); + labels +} + +/// Per series (without `__name__`), the sum of `workload`'s samples in +/// `(at_ms - window_ms, at_ms]`, computed from the regenerated samples. +fn sums_over_window( + workload: &WorkloadSpec, + window_ms: u64, + at_ms: i64, +) -> BTreeMap { + let mut sums = BTreeMap::new(); + for chunk in workload_chunks(&workload.data, 10_000).unwrap() { + for sample in chunk.unwrap().samples { + if sample.ts_ms > at_ms - window_ms as i64 && sample.ts_ms <= at_ms { + *sums.entry(without_name(&sample.labels)).or_insert(0.0) += sample.value; + } + } + } + sums +} + +fn arm_result(run: &RunDir, workload: &str, variant: &str) -> ArmResult { + let path = run + .path() + .join("results/asap") + .join(format!("{workload}__{variant}.json")); + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +/// `sum_over_time(m[5m])` every 10 s over 2 series, planned twice: with +/// samples arriving (a chain of 10 s tumbling windows built at ingestion +/// time) and at rest (raw samples scanned at query time). The ASAP lane +/// runs both plans (one warm-up and three measured runs, so four installs +/// interleaved) and answers every scheduled evaluation exactly: per series, +/// the sum of its samples in `(T - 5 min, T]`. +#[test] +fn the_asap_lane_runs_a_planned_sum_over_time_and_answers_exactly() { + let temp = TempDir::new("sum"); + let workload = harness_workload("w", SUM_QUERY, 300_000); + let ingesting = temp.0.join("ingesting.plan.json"); + let at_rest = temp.0.join("at-rest.plan.json"); + for (plan_file, arrival) in [ + (&ingesting, r#"{"ingesting": {"rows_per_second": 1000.0}}"#), + (&at_rest, r#""at_rest""#), + ] { + let planning = planning_workload(SUM_QUERY, "m", 1_000, 1_000, arrival); + write_plan_file(&planning, plan_file); + } + let run = run_dir( + &temp, + &[&workload], + &[("ingesting", "w", ingesting), ("at-rest", "w", at_rest)], + ); + let cells = run_asap_lane(&run, 1, 3); + assert_eq!(cells.len(), 2, "{cells:?}"); + assert!( + cells.iter().all(|cell| cell.outcome == CellOutcome::Ran), + "{cells:?}" + ); + + let schedule = ArmSchedule::for_workload(&workload).unwrap(); + for variant in ["ingesting", "at-rest"] { + let result = arm_result(&run, "w", variant); + assert!(result.plan.substitutions.is_empty()); + assert!(result.plan.sharing_lost.is_empty()); + assert!(!result.plan.node_kinds.is_empty()); + let times: Vec = result.answers.iter().map(|(at_ms, _)| *at_ms).collect(); + assert_eq!(times, schedule.evaluate_at_ms, "{variant}"); + for (at_ms, answers) in &result.answers { + let expected = sums_over_window(&workload, 300_000, *at_ms); + assert_eq!(expected.len(), 2); + let answered = &answers.0["sum"]; + assert_eq!( + answered.keys().collect::>(), + expected.keys().collect::>(), + "{variant} at {at_ms}" + ); + for (series, value) in &expected { + let got = answered[series]; + assert!( + (got - value).abs() <= 1e-9 * value.abs(), + "{variant} at {at_ms} {series:?}: {got} vs {value}" + ); + } + } + // The plan's shape, a precondition: if the planner changes it, pick + // workloads that keep one ingestion-time and one query-time plan. + match variant { + "ingesting" => { + assert!(!result.plan.stored_outputs.is_empty(), "{:?}", result.plan); + assert!(result.footprint.view_bytes > 0, "{:?}", result.footprint); + } + _ => { + assert!(result.plan.stored_outputs.is_empty(), "{:?}", result.plan); + assert!( + result.footprint.raw_retained_bytes > 0, + "{:?}", + result.footprint + ); + } + } + } +} + +/// `absent_over_time(m[10s])`: Stages 1-3 plan it, but the Planner's +/// executor cannot compile it. `asap_planner` still writes the plan file +/// (with a note), and on the ASAP lane it is a "not executable" cell, so +/// the other lanes still run that plan. +#[test] +fn a_plan_the_executor_cannot_compile_is_a_not_executable_cell() { + const QUERY: &str = "absent_over_time(m[10s])"; + let temp = TempDir::new("absent"); + let planning = planning_workload(QUERY, "m", 2, 1_000, r#""at_rest""#); + let plan_file = temp.0.join("absent.plan.json"); + // Precondition: an executor-only gap. Once the executor runs + // `absent_over_time`, pick another query it plans but cannot compile. + let stderr = write_plan_file(&planning, &plan_file); + assert!( + stderr.contains("note: the Planner's executor cannot run this plan: "), + "{stderr}" + ); + let workload = harness_workload("w", QUERY, 10_000); + let run = run_dir(&temp, &[&workload], &[("exact", "w", plan_file)]); + let cells = run_asap_lane(&run, 0, 1); + let [cell] = cells.as_slice() else { + panic!("{cells:?}") + }; + match &cell.outcome { + CellOutcome::NotRun { error } => { + assert!(error.starts_with("not executable: "), "{error}") + } + other => panic!("{other:?}"), + } +} + +/// `install`'s refusals: no plan file in the engine options, a missing file, +/// a file that is not a plan, and a plan for other queries (query `i` is +/// answered by the plan's root `i`, matched by text in any of the query's +/// languages). +#[test] +fn a_plan_for_other_queries_is_refused_naming_the_query() { + let temp = TempDir::new("refusals"); + let planning = planning_workload(SUM_QUERY, "m", 2, 1_000, r#""at_rest""#); + let plan_file = temp.0.join("sum.plan.json"); + write_plan_file(&planning, &plan_file); + let arm = |options: serde_json::Value| ArmSpec { + name: "exact".into(), + raw_data_retained: true, + engine_options: options, + }; + let with_plan = arm(serde_json::json!({ "plan_file": plan_file })); + let install_error = + |workload: &WorkloadSpec, arm: &ArmSpec| match AsapEngine.install(workload, arm) { + Ok(_) => panic!("installed"), + Err(EngineError(message)) => message, + }; + let workload = harness_workload("w", SUM_QUERY, 300_000); + + let error = install_error(&workload, &arm(serde_json::Value::Null)); + assert!(error.contains("plan_file"), "{error}"); + let missing = temp.0.join("missing.plan.json"); + let error = install_error(&workload, &arm(serde_json::json!({ "plan_file": missing }))); + assert!(error.contains("missing.plan.json"), "{error}"); + let not_a_plan = temp.0.join("not-a-plan.json"); + std::fs::write(¬_a_plan, "{}").unwrap(); + let error = install_error( + &workload, + &arm(serde_json::json!({ "plan_file": not_a_plan })), + ); + assert!(error.contains("not-a-plan.json"), "{error}"); + + // Another query text: the error names the query and both texts. + let other = harness_workload("w", "count_over_time(m[5m])", 300_000); + let error = install_error(&other, &with_plan); + for part in ["query 0", "sum", SUM_QUERY, "count_over_time(m[5m])"] { + assert!(error.contains(part), "{part}: {error}"); + } + // Two queries against a plan with one root. + let mut two = workload.clone(); + two.queries.push(WorkloadQuery { + id: "second".into(), + ..two.queries[0].clone() + }); + let error = install_error(&two, &with_plan); + assert!( + error.contains("1 query text") && error.contains("2 queries"), + "{error}" + ); + + // The plan's text as the query's second text (after a SQL text) matches. + let mut sql_first = workload.clone(); + sql_first.queries[0].texts.insert( + 0, + QueryText { + language: QueryLanguage::Sql, + text: "SELECT s, sum(value) FROM m GROUP BY s".into(), + }, + ); + assert!(AsapEngine.install(&sql_first, &with_plan).is_ok()); +} + +/// `asap_run bench-external --lane asap` runs the lane over a run directory +/// and writes each cell; an unknown lane is a usage error naming the known +/// one. +#[test] +fn asap_run_runs_the_asap_lane_over_a_run_dir() { + let temp = TempDir::new("asap-run"); + let workload = harness_workload("w", SUM_QUERY, 300_000); + let plan_file = temp.0.join("sum.plan.json"); + let planning = planning_workload(SUM_QUERY, "m", 2, 1_000, r#""at_rest""#); + write_plan_file(&planning, &plan_file); + let run = run_dir(&temp, &[&workload], &[("exact", "w", plan_file)]); + let asap_run = |lane: &str| { + Command::new(env!("CARGO_BIN_EXE_asap_run")) + .args(["bench-external", "--lane", lane]) + .args(["--run-dir", run.path().to_str().unwrap()]) + .args(["--core", &allowed_core().to_string()]) + .args(["--warmup", "0", "--runs", "1", "--max-core-busy", "1.0"]) + .output() + .unwrap() + }; + let output = asap_run("asap"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let cell: aqpbm_core::harness::bench_external::LaneCell = serde_json::from_str( + &std::fs::read_to_string(run.path().join("results/asap/w__exact.cell.json")).unwrap(), + ) + .unwrap(); + assert_eq!(cell.outcome, CellOutcome::Ran); + + let output = asap_run("fake"); + assert_eq!(output.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&output.stderr).contains("asap")); +} diff --git a/crates/devtools/tests/asap_lane_conformance.rs b/crates/devtools/tests/asap_lane_conformance.rs new file mode 100644 index 000000000..645b44930 --- /dev/null +++ b/crates/devtools/tests/asap_lane_conformance.rs @@ -0,0 +1,92 @@ +//! The harness's conformance kit on the ASAP lane. It checks that the +//! executor's work runs on the calling thread by comparing thread and process +//! CPU, so it runs in its own test binary: other tests running in the same +//! process would add process CPU the lane did not spend. +use std::collections::BTreeMap; +use std::path::PathBuf; + +use aqpbm_core::harness::conformance::{self, ConformanceCheck}; +use aqpbm_core::harness::engine::{ + ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, + QueryLanguage, WorkloadSpec, +}; +use aqpbm_core::harness::protocol::{MeasurementClocks, OsCpuClock}; +use asap_devtools::asap_lane::AsapEngine; +use asap_planner::{plan_installable, FrontendInput, PlanningModels, PlanningWorkload, UserInput}; + +#[path = "asap_lane_support.rs"] +mod support; +use support::{planning_workload, TempDir}; + +/// The conformance kit's probe arm has no engine options, so this engine +/// hands `AsapEngine` the plan file of each probe workload, by name. +struct WithPlanFiles { + plan_files: BTreeMap, +} + +impl EngineUnderTest for WithPlanFiles { + fn description(&self) -> EngineDescription { + AsapEngine.description() + } + fn cpu_accounting(&self) -> CpuAccounting { + AsapEngine.cpu_accounting() + } + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let arm = ArmSpec { + engine_options: serde_json::json!({"plan_file": self.plan_files[&workload.name]}), + ..arm.clone() + }; + AsapEngine.install(workload, &arm) + } +} + +/// The harness's conformance kit passes on the ASAP lane: deterministic +/// answers, left-open windows (`count_over_time(probe[10s])` at 20 s counts +/// only the sample at 20 s), and the executor's work on the calling thread. +#[test] +fn conformance_kit_passes_on_the_asap_engine() { + let temp = TempDir::new("conformance"); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let mut plan_files = BTreeMap::new(); + for workload in conformance::probe_workloads() { + let query = &workload.queries[0].texts[0]; + assert_eq!(query.language, QueryLanguage::PromQl); + let planning: PlanningWorkload = serde_json::from_str(&planning_workload( + &query.text, + "probe", + 1, + 10_000, + r#"{"ingesting": {"rows_per_second": 0.1}}"#, + )) + .unwrap(); + let (plan, _) = runtime + .block_on(plan_installable(UserInput::new( + &planning, + FrontendInput::Promql, + PlanningModels::builtin(), + ))) + .unwrap(); + let path = temp.0.join(format!("{}.plan.json", workload.name)); + std::fs::write(&path, serde_json::to_string(&plan).unwrap()).unwrap(); + plan_files.insert(workload.name.clone(), path); + } + let mut engine = WithPlanFiles { plan_files }; + let report = conformance::check( + &mut engine, + &MeasurementClocks::with_calling_thread(&OsCpuClock), + ) + .unwrap_or_else(|failures| panic!("{failures:#?}")); + for check in [ + ConformanceCheck::DeterministicAnswers, + ConformanceCheck::LeftOpenWindow, + ConformanceCheck::WorkOnCallingThread, + ] { + assert!(report.passed.contains(&check), "{report:?}"); + } +} diff --git a/crates/devtools/tests/asap_lane_support.rs b/crates/devtools/tests/asap_lane_support.rs new file mode 100644 index 000000000..ad6f2faba --- /dev/null +++ b/crates/devtools/tests/asap_lane_support.rs @@ -0,0 +1,45 @@ +//! Helpers shared by the ASAP lane tests (`asap_lane.rs`, in the `devtools` +//! test binary) and the conformance test, which runs in its own binary. +use std::path::PathBuf; + +/// A directory under the build's test temp directory, removed when dropped. +pub struct TempDir(pub PathBuf); + +impl TempDir { + pub fn new(test: &str) -> Self { + let path = PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(format!("asap-lane-{}-{test}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + TempDir(path) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// The planning workload of one exact PromQL `query` evaluated every 10 s at +/// phase 0 over gauge `metric` with `series_count` series sampled every +/// `sample_interval_ms`, arriving as `arrival` (`{"ingesting": ...}` or +/// `"at_rest"`). +pub fn planning_workload( + query: &str, + metric: &str, + series_count: u64, + sample_interval_ms: u64, + arrival: &str, +) -> String { + format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ + "query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {{"max_error": 0.0, "failure_probability": 0.0}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {arrival}, + "sources": {{{metric:?}: {{"series_count": {series_count}, "metric_type": "gauge", + "sample_interval": {sample_interval_ms}}}}}}}}}"# + ) +} diff --git a/crates/devtools/tests/main.rs b/crates/devtools/tests/main.rs index ed12546d0..82c45c766 100644 --- a/crates/devtools/tests/main.rs +++ b/crates/devtools/tests/main.rs @@ -2,6 +2,7 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-devtools --test devtools ::`. +mod asap_lane; mod asap_planner; mod asap_planner_acceptance; mod cross_language; diff --git a/crates/executor/src/sources/memory.rs b/crates/executor/src/sources/memory.rs index ae4196444..14fd571b3 100644 --- a/crates/executor/src/sources/memory.rs +++ b/crates/executor/src/sources/memory.rs @@ -2,28 +2,55 @@ use super::*; use planner_types::plan::physical_asap_dag::PhysicalASAPNodeId; use std::collections::BTreeMap; -/// An in-memory [`DataStore`], for tests and examples: raw tables by -/// `Source`, and each materialized view's rows by its node id, time-indexed -/// and append-only. A run's output for a view is appended with the time -/// range its rows cover: a chain's window `(window_end - w, window_end]`, -/// or the range of data another view's state summarizes. Appending drops the -/// rows older than the view's `retention_ms` before the newest appended -/// end. A read returns the rows whose time ranges tile the requested range -/// exactly, and fails when one is missing. Raw tables return every row (the -/// executor re-applies the raw data request). +/// An in-memory [`DataStore`]: raw tables by `Source`, and each +/// materialized view's rows by its node id, time-indexed and append-only. +/// A run's output for a view is appended with the time range its rows +/// cover: a chain's window `(window_end - w, window_end]`, or the range of +/// data another view's state summarizes. Appending drops the rows older +/// than the view's `retention_ms` before the newest appended end. A read +/// returns the rows whose time ranges tile the requested range exactly, and +/// fails when one is missing. +/// +/// A raw table is bound whole ([`Self::with_raw_data`]): every read returns +/// every row; or bound for appending ([`Self::with_appended_raw_data`]): a +/// runtime appends each batch of samples with the `(start, end]` it covers +/// as it arrives, a read returns the batches overlapping the requested +/// time range, and appending drops the batches older than the table's +/// lookback. Either way the executor re-applies the raw data request. /// /// The store owns the resident batches (a run is not charged for their /// states again); each cursor clones only the next batch, not the entire /// data set. #[derive(Default)] pub struct InMemoryDataStore { - raw_data: Vec<(Source, SchemaRef, Vec)>, + raw_data: Vec, /// Each view's appended rows, oldest first. materialized_views: BTreeMap>, } -/// Rows appended for a view: the `(start, end]` they cover and their batches. +/// Rows appended for a view or a raw table bound for appending: the +/// `(start, end]` they cover and their batches. type AppendedRows = ((i64, i64), Vec); + +/// The rows of one raw table. +struct RawTable { + source: Source, + schema: SchemaRef, + rows: RawRows, +} + +enum RawRows { + /// Bound whole: every read returns every row. + Whole(Vec), + /// Appended as samples arrive, oldest first; appending drops the rows + /// ending `lookback_ms` or more before the newest end (`None`: kept + /// forever). + Appended { + lookback_ms: Option, + rows: Vec, + }, +} + impl InMemoryDataStore { /// Bind `batches`, all in `schema`, as the rows of `source`. Bad input /// when `source` is already bound or a batch is in another schema. @@ -33,19 +60,120 @@ impl InMemoryDataStore { schema: SchemaRef, batches: Vec, ) -> Result { - crate::values::validate_schema(&schema)?; - if self.raw_data.iter().any(|(bound, _, _)| bound == &source) { - return Err(Error::Invalid(format!( - "raw data {source:?} is bound twice" - ))); - } if batches.iter().any(|batch| batch.schema() != &schema) { return Err(Error::Invalid("raw data batch schema mismatch".into())); } held_by_store(&batches); - self.raw_data.push((source, schema, batches)); + self.bind_raw_data(source, schema, RawRows::Whole(batches))?; Ok(self) } + + /// Bind `source`, in `schema`, for appending its rows as they arrive + /// ([`Self::append_raw_data`]), keeping each batch until the newest + /// appended end is `lookback_ms` past its end (`None`: forever). Bad + /// input when `source` is already bound. Example: a runtime keeping + /// the last 5 minutes of a time series every query-time `Scan` of it + /// reads binds it with `Some(300_000)`. + pub fn with_appended_raw_data( + mut self, + source: Source, + schema: SchemaRef, + lookback_ms: Option, + ) -> Result { + let rows = RawRows::Appended { + lookback_ms, + rows: Vec::new(), + }; + self.bind_raw_data(source, schema, rows)?; + Ok(self) + } + + /// Append `batches`, all in the table's schema, as the rows of `source` + /// covering `time_range_ms` (`(start, end]`), then drop its batches + /// ending `lookback_ms` or more before `end` (the newest are always + /// kept). Bad input when `source` is not bound for appending, a batch + /// is in another schema, or the range is empty or starts before the end + /// of the rows appended last (the store is append-only). + pub fn append_raw_data( + &mut self, + source: &Source, + time_range_ms: (i64, i64), + batches: Vec, + ) -> Result<(), Error> { + let table = self + .raw_data + .iter_mut() + .find(|table| &table.source == source) + .ok_or_else(|| Error::Invalid(format!("unbound raw data: {source:?}")))?; + let RawRows::Appended { lookback_ms, rows } = &mut table.rows else { + return Err(Error::Invalid(format!( + "raw data {source:?} is bound whole, not for appending" + ))); + }; + if batches.iter().any(|batch| batch.schema() != &table.schema) { + return Err(Error::Invalid("raw data batch schema mismatch".into())); + } + let (start, end) = time_range_ms; + if start >= end + || rows + .last() + .is_some_and(|((_, last_end), _)| start < *last_end) + { + return Err(Error::Invalid(format!( + "raw data {source:?}: appended rows ({start}, {end}] are empty or not after the stored ones" + ))); + } + held_by_store(&batches); + rows.push((time_range_ms, batches)); + if let Some(lookback_ms) = *lookback_ms { + drop_rows_ending_before(rows, end, lookback_ms); + } + Ok(()) + } + + /// Bytes of the raw rows held. + pub fn raw_data_bytes(&self) -> usize { + self.raw_data + .iter() + .map(|table| match &table.rows { + RawRows::Whole(batches) => batches_bytes(batches), + RawRows::Appended { rows, .. } => { + rows.iter().map(|(_, batches)| batches_bytes(batches)).sum() + } + }) + .sum() + } + + /// Bytes of the materialized views' stored rows. + pub fn materialized_view_bytes(&self) -> usize { + self.materialized_views + .values() + .flatten() + .map(|(_, batches)| batches_bytes(batches)) + .sum() + } + + /// Bind `source` to `rows`, all in `schema`; bad input when it is + /// already bound. + fn bind_raw_data( + &mut self, + source: Source, + schema: SchemaRef, + rows: RawRows, + ) -> Result<(), Error> { + crate::values::validate_schema(&schema)?; + if self.raw_data.iter().any(|table| table.source == source) { + return Err(Error::Invalid(format!( + "raw data {source:?} is bound twice" + ))); + } + self.raw_data.push(RawTable { + source, + schema, + rows, + }); + Ok(()) + } /// Append `batches` as the rows of `view` covering `time_range_ms` /// (`(start, end]`), then drop its rows ending `view.retention_ms()` or /// more before `end` (the newest rows are always kept). Bad input when @@ -71,13 +199,23 @@ impl InMemoryDataStore { } held_by_store(&batches); rows.push((time_range_ms, batches)); - let oldest_kept_end = - end.saturating_sub(i64::try_from(view.retention_ms()).unwrap_or(i64::MAX)); - rows.retain(|((_, row_end), _)| *row_end > oldest_kept_end || *row_end == end); + drop_rows_ending_before(rows, end, view.retention_ms()); Ok(()) } } +/// Drops the rows of `rows` ending `retention_ms` or more before `end`, +/// keeping those ending at `end`: retention 15 at end 30 drops `(0, 10]` +/// and keeps `(10, 20]`. +fn drop_rows_ending_before(rows: &mut Vec, end: i64, retention_ms: u64) { + let oldest_kept_end = end.saturating_sub(i64::try_from(retention_ms).unwrap_or(i64::MAX)); + rows.retain(|((_, row_end), _)| *row_end > oldest_kept_end || *row_end == end); +} + +fn batches_bytes(batches: &[Batch]) -> usize { + batches.iter().map(Batch::memory_bytes).sum() +} + fn held_by_store(batches: &[Batch]) { for batch in batches { batch.mark_states_held_outside_run(); @@ -108,17 +246,28 @@ impl DataStore for InMemoryDataStore { request: &RawDataRequest, context: RunContext, ) -> Result, Error> { - let (_, schema, batches) = self + let table = self .raw_data .iter() - .find(|(bound, _, _)| bound == source) + .find(|table| &table.source == source) .ok_or_else(|| Error::Invalid(format!("unbound raw data: {source:?}")))?; - let projection = projection(schema, request.schema)?; + let projection = projection(&table.schema, request.schema)?; let output = request.schema.clone(); + // An appended table returns the batches overlapping the requested + // range; the executor re-applies it row by row. + let batches: Box> = match (&table.rows, request.time_range_ms) { + (RawRows::Whole(batches), _) => Box::new(batches.iter()), + (RawRows::Appended { rows, .. }, None) => { + Box::new(rows.iter().flat_map(|(_, batches)| batches)) + } + (RawRows::Appended { rows, .. }, Some((start, end))) => Box::new( + rows.iter() + .filter(move |((row_start, row_end), _)| *row_end > start && *row_start < end) + .flat_map(|(_, batches)| batches), + ), + }; Ok(stream_batches( - batches - .iter() - .map(move |batch| batch.project(output.clone(), &projection)), + batches.map(move |batch| batch.project(output.clone(), &projection)), context, )) } diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 192d63371..10561b58b 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -994,3 +994,115 @@ fn a_materialized_view_is_requested_for_the_window_it_summarizes() { ); } } + +/// Rows `(ts, value)` with one row at each of `times`, in `schema`. +fn rows_at(schema: &SchemaRef, times: &[i64]) -> Batch { + Batch::try_new( + schema.clone(), + times + .iter() + .map(|&ts| vec![Value::Timestamp(ts), Value::Float64(ts as f64)]) + .collect(), + ) + .unwrap() +} + +/// The times of the rows `store` returns for `source` read over `range`. +fn times_read( + store: &InMemoryDataStore, + source: &Source, + schema: &SchemaRef, + range: Option<(i64, i64)>, +) -> Vec { + let request = RawDataRequest { + time_range_ms: range, + schema, + predicates: &[], + }; + let batches: Vec = block_on( + store + .read_raw_data(source, &request, context()) + .unwrap() + .map(Result::unwrap) + .collect(), + ); + batches + .iter() + .flat_map(Batch::to_rows) + .map(|row| match row[0] { + Value::Timestamp(ts) => ts, + ref other => panic!("{other:?}"), + }) + .collect() +} + +/// A runtime appends a time series' raw rows as they arrive, each batch with +/// the `(start, end]` it covers: a read returns the batches overlapping the +/// requested range (the executor re-applies the range), and appending drops +/// the batches ending `lookback_ms` or more before the newest end. E.g. +/// lookback 15 ms at end 30 drops `(0, 10]` and keeps `(10, 20]`, which +/// holds rows a window `(15, 30]` reads. +#[test] +fn appended_raw_data_is_read_by_range_and_dropped_after_its_lookback() { + let schema = Arc::new(planner_types::ir::schema::Schema { + unique_keys: vec![], + closed: false, + fields: vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + time_index: Some(0), + }); + let source = Source::TimeSeries { metric: "m".into() }; + let mut store = InMemoryDataStore::default() + .with_appended_raw_data(source.clone(), schema.clone(), Some(15)) + .unwrap(); + assert_eq!(store.raw_data_bytes(), 0); + for (start, end) in [(0, 10), (10, 20)] { + store + .append_raw_data(&source, (start, end), vec![rows_at(&schema, &[end])]) + .unwrap(); + } + assert_eq!( + times_read(&store, &source, &schema, Some((5, 20))), + [10, 20] + ); + assert_eq!(times_read(&store, &source, &schema, Some((10, 20))), [20]); + store + .append_raw_data(&source, (20, 30), vec![rows_at(&schema, &[25, 30])]) + .unwrap(); + assert_eq!(times_read(&store, &source, &schema, None), [20, 25, 30]); + assert_eq!( + times_read(&store, &source, &schema, Some((15, 30))), + [20, 25, 30] + ); + assert_eq!( + times_read(&store, &source, &schema, Some((20, 30))), + [25, 30] + ); + let held = rows_at(&schema, &[20]).memory_bytes() + rows_at(&schema, &[25, 30]).memory_bytes(); + assert_eq!(store.raw_data_bytes(), held); + assert_eq!(store.materialized_view_bytes(), 0); + + // Append-only, to a source bound for appending. + assert!(store + .append_raw_data(&source, (25, 40), vec![rows_at(&schema, &[40])]) + .is_err()); + let other = Source::TimeSeries { + metric: "other".into(), + }; + assert!(store + .append_raw_data(&other, (30, 40), vec![rows_at(&schema, &[40])]) + .is_err()); + let fixed = Source::TimeSeries { + metric: "fixed".into(), + }; + let mut store = store + .with_raw_data(fixed.clone(), schema.clone(), vec![rows_at(&schema, &[1])]) + .unwrap(); + assert!(store + .append_raw_data(&fixed, (30, 40), vec![rows_at(&schema, &[40])]) + .is_err()); + // Rows bound whole are returned whatever the range. + assert_eq!(times_read(&store, &fixed, &schema, Some((20, 30))), [1]); +} diff --git a/crates/planner/src/installable/mod.rs b/crates/planner/src/installable/mod.rs index 5290f93ad..a7f9a77bf 100644 --- a/crates/planner/src/installable/mod.rs +++ b/crates/planner/src/installable/mod.rs @@ -26,6 +26,7 @@ use std::collections::{HashMap, HashSet}; use asap_cost::{NodeCostSource, OperatorKind, PlanCost}; +use asap_executor::physical_planner::CompiledPhysicalDAG; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::plan::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, @@ -308,6 +309,40 @@ impl InstallablePlan { executor_compiles(&self.dag, &views).map_err(|error| error.to_string()) } + /// The executor's query run of this plan, as a runtime runs it at each + /// evaluation: the roots, in workload order, over the views they read + /// (each a materialized view slot of the runtime's store) and the + /// `Scan`s they read outside them (raw data slots). `Err` is + /// [`InstallablePlanError::QueryRunNotExecutable`] when the executor + /// cannot compile it, or why the views cannot be derived. + pub fn compile_query_run(&self) -> Result { + let views = self.materialized_views()?; + let views_by_node = views.iter().map(|view| (view.node_id, view)).collect(); + executor_runs_queries(&IndexedDag::new(&self.dag), &views_by_node) + } + + /// The executor's refresh of `group`, one of + /// [`Self::ingestion_refresh_groups`], as a runtime runs it in one + /// ingestion run per window: the group's views as roots, in group + /// order, over the nodes below them, each view of another group they + /// read a materialized view slot. `Err` is + /// [`InstallablePlanError::MaterializedViewNotExecutable`] when the + /// executor cannot compile it, or why the views cannot be derived. + pub fn compile_ingestion_refresh( + &self, + group: &IngestionRefreshGroup, + ) -> Result { + let views = self.materialized_views()?; + let views_by_node = views.iter().map(|view| (view.node_id, view)).collect(); + let refreshed: Vec<&MaterializedView> = group.views.iter().collect(); + executor_refreshes( + &refreshed, + &views_by_node, + &IndexedDag::new(&self.dag), + false, + ) + } + /// The plan of `dag` answering `query_texts` (one per root, in workload /// order), checked as [`Self::validate`] checks a plan file: the DAG's /// validity, one text per root, no standalone scalar query, and each @@ -474,7 +509,7 @@ fn executor_compiles( for (position, refreshed) in refreshes.enumerate() { executor_refreshes(&refreshed, &views_by_node, &indexed, position == 0)?; } - executor_runs_queries(&indexed, &views_by_node) + executor_runs_queries(&indexed, &views_by_node).map(|_| ()) } /// Whether the executor compiles the query run: the roots over the views @@ -484,7 +519,7 @@ fn executor_compiles( fn executor_runs_queries( indexed: &IndexedDag<'_>, views_by_node: &HashMap, -) -> Result<(), InstallablePlanError> { +) -> Result { use asap_executor::physical_planner::compile; let dag = indexed.dag; let mut read = Vec::new(); @@ -500,10 +535,8 @@ fn executor_runs_queries( } } let roots: Vec = dag.roots.iter().map(|&id| id as u64).collect(); - compile(dag, &read, &roots).map(|_| ()).map_err(|error| { - InstallablePlanError::QueryRunNotExecutable { - reason: error.to_string(), - } + compile(dag, &read, &roots).map_err(|error| InstallablePlanError::QueryRunNotExecutable { + reason: error.to_string(), }) } @@ -516,7 +549,7 @@ fn executor_refreshes( views_by_node: &HashMap, indexed: &IndexedDag<'_>, in_whole_plan: bool, -) -> Result<(), InstallablePlanError> { +) -> Result { use asap_executor::physical_planner::compile; let roots: Vec = refreshed.iter().map(|view| view.node_id).collect(); // The refresh reads only the nodes below it: compile them, or the whole @@ -541,17 +574,15 @@ fn executor_refreshes( .filter_map(|id| views_by_node.get(id).map(|read| (*read).clone())) .collect(); let root_ids: Vec = roots.iter().map(|&id| id as u64).collect(); - compile(below, &read, &root_ids) - .map(|_| ()) - .map_err( - |error| InstallablePlanError::MaterializedViewNotExecutable { - node_id: roots[0], - reason: match roots.as_slice() { - [_] => error.to_string(), - _ => format!("refreshed with views {roots:?}: {error}"), - }, + compile(below, &read, &root_ids).map_err(|error| { + InstallablePlanError::MaterializedViewNotExecutable { + node_id: roots[0], + reason: match roots.as_slice() { + [_] => error.to_string(), + _ => format!("refreshed with views {roots:?}: {error}"), }, - ) + } + }) } /// The data facts the selected `plan` (costed as `cost`) assumed because diff --git a/crates/planner/tests/time_series/mod.rs b/crates/planner/tests/time_series/mod.rs index 0be819d9b..2f808444f 100644 --- a/crates/planner/tests/time_series/mod.rs +++ b/crates/planner/tests/time_series/mod.rs @@ -17,8 +17,7 @@ use asap_executor::Error; use asap_planner::{InstallablePlan, MaterializedView}; use asap_types::ir::operator::Source; use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; -use asap_types::ir::NonASAPOp; -use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use futures::{executor::block_on, StreamExt}; /// One raw data sample as it arrives, e.g. one Prometheus Remote Write sample. @@ -67,40 +66,24 @@ pub fn series_identity(labels: &BTreeMap) -> String { /// every column those `Scan`s read. pub fn time_series_tables(dag: &PhysicalASAPDAG, samples: &[RawDataSample]) -> InMemoryDataStore { // Each metric's columns: the union of its Scans' columns, by name. - let mut schemas: BTreeMap<&str, Schema> = BTreeMap::new(); - for node in &dag.nodes { - let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric }, - schema, - .. - }) = &node.payload - else { - continue; - }; - let known = schemas.entry(metric).or_insert_with(|| schema.clone()); - for field in &schema.fields { - if !known.fields.iter().any(|f| f.name == field.name) { - known.fields.push(field.clone()); - } - } - } + let schemas = dag + .raw_data_schemas() + .into_iter() + .filter_map(|(source, schema)| match source { + Source::TimeSeries { metric } => Some((metric, schema)), + Source::Table { .. } => None, + }); let mut tables = InMemoryDataStore::default(); for (metric, schema) in schemas { let schema = Arc::new(schema); let rows = samples .iter() - .filter(|s| s.labels.get("__name__").map(String::as_str) == Some(metric)) + .filter(|s| s.labels.get("__name__") == Some(&metric)) .map(|sample| row(&schema, sample)) .collect(); let batch = Batch::try_new(schema.clone(), rows).unwrap(); tables = tables - .with_raw_data( - Source::TimeSeries { - metric: metric.to_string(), - }, - schema, - vec![batch], - ) + .with_raw_data(Source::TimeSeries { metric }, schema, vec![batch]) .unwrap(); } tables diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index e3a8182ae..53476da6d 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -242,7 +242,9 @@ impl RawDataRetention { lookback_ms, } } +} +impl RawDataRetention { /// The source. pub fn source(&self) -> &Source { &self.source diff --git a/crates/types/src/plan/materialized_view/raw_data.rs b/crates/types/src/plan/materialized_view/raw_data.rs index b17f8ac1e..30e2c8a27 100644 --- a/crates/types/src/plan/materialized_view/raw_data.rs +++ b/crates/types/src/plan/materialized_view/raw_data.rs @@ -1,13 +1,16 @@ //! How far back a runtime keeps each raw-data source for the DAG's raw-data //! scans ([`PhysicalASAPDAG::raw_data_retention`]), which nodes keep rows -//! ([`PhysicalASAPDAGNode::retains_rows`]), and the shape checks of every -//! raw-data window ([`PhysicalASAPDAG::check_raw_data_windows`]). +//! ([`PhysicalASAPDAGNode::retains_rows`]), the shape checks of every +//! raw-data window ([`PhysicalASAPDAG::check_raw_data_windows`]), and the +//! columns a runtime keeps of each source ([`PhysicalASAPDAG::raw_data_schemas`]). use super::views::ViewIndex; use super::{InstallableDagError, RawDataRetention}; +use crate::ir::operator::arguments::Source; use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; +use crate::ir::schema::Schema; use crate::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId}; use crate::stages::Lifetime; @@ -55,6 +58,31 @@ impl PhysicalASAPDAG { Ok(retention) } + /// Each raw-data source the DAG's `Scan`s read (at query or ingestion + /// time), in the order of its first `Scan`, with the union of those + /// `Scan`s' columns by name, in first-seen order: the raw table a + /// runtime binds for the source. Example: `sum by (job) (m)` and + /// `sum_over_time(m[5m])` together read `m` as `(ts, value, + /// $promql_series_identity, job)`. + pub fn raw_data_schemas(&self) -> Vec<(Source, Schema)> { + let mut schemas: Vec<(Source, Schema)> = Vec::new(); + for node in &self.nodes { + let Operator::NonASAP(NonASAPOp::Scan { source, schema, .. }) = &node.payload else { + continue; + }; + let Some((_, known)) = schemas.iter_mut().find(|(known, _)| known == source) else { + schemas.push((source.clone(), schema.clone())); + continue; + }; + for field in &schema.fields { + if !known.fields.iter().any(|known| known.name == field.name) { + known.fields.push(field.clone()); + } + } + } + schemas + } + /// Checks that every raw-data window the DAG reads (each `TimeRange`, /// and each `SummaryTumblingWindowChain`'s tumbling window, reading a /// `Scan` without passing another `TimeRange`) has a supported shape