From f6ca79420a03e3fc914038f635a6c812a7be4e10 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:24:51 +0000 Subject: [PATCH 1/7] test: the in-process ASAP lane of the engine comparison (failing first) AsapEngine installs a plan file written by asap_planner --plan-file and runs it with the Planner's executor under the sketch-bench harness: the harness's conformance kit, sum_over_time answered exactly from an ingestion-time chain and from raw samples at query time, a plan the executor cannot compile as a 'not executable' cell, install refusals, and asap_run bench-external --lane asap. The executor's in-memory store gains appended, time-ranged raw data dropped after its lookback. --- Cargo.lock | 64 ++++ crates/devtools/Cargo.toml | 8 + crates/devtools/tests/asap_lane.rs | 510 +++++++++++++++++++++++++++++ crates/devtools/tests/main.rs | 1 + crates/executor/tests/raw_scan.rs | 99 ++++++ 5 files changed, 682 insertions(+) create mode 100644 crates/devtools/tests/asap_lane.rs diff --git a/Cargo.lock b/Cargo.lock index 703605ad6..c6ee11bb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,48 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "aqpbm-core" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=af7fa57c917696815937f7cdd84f48c3391e1c89#af7fa57c917696815937f7cdd84f48c3391e1c89" +dependencies = [ + "anyhow", + "aqpbm-datagen", + "bitflags", + "chrono", + "libc", + "rand 0.9.5", + "rand_xoshiro", + "serde", + "serde_json", +] + +[[package]] +name = "aqpbm-datagen" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=af7fa57c917696815937f7cdd84f48c3391e1c89#af7fa57c917696815937f7cdd84f48c3391e1c89" +dependencies = [ + "rand 0.9.5", + "rand_distr", + "rand_xoshiro", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "aqpbm-harness" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=af7fa57c917696815937f7cdd84f48c3391e1c89#af7fa57c917696815937f7cdd84f48c3391e1c89" +dependencies = [ + "aqpbm-core", + "aqpbm-datagen", + "chrono", + "libc", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -306,6 +348,8 @@ dependencies = [ name = "asap-devtools" version = "0.1.0" dependencies = [ + "aqpbm-datagen", + "aqpbm-harness", "asap-cost", "asap-executor", "asap-frontend-metricsql", @@ -318,6 +362,7 @@ dependencies = [ "asap-stage2-physical-optimization", "asap-stage3-plan-selection", "asap-types", + "futures", "serde", "serde_json", "serde_yaml", @@ -2582,6 +2627,25 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.5", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "recursive" version = "0.1.1" diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 37c4ff86d..a000cb2d3 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -20,6 +20,12 @@ asap-planner = { path = "../planner" } # The executor's supported summary families, which `asap_planner` plans # with when the deployment declares none, as `plan_installable` does. asap-executor = { path = "../executor" } +# The engine-comparison harness of sketch-bench: `asap_run bench-external +# --lane asap` runs the Planner's executor as an engine under test +# (`asap_lane::AsapEngine`). Pinned to a sketch-bench commit; sketch-bench +# never depends on the planner. +aqpbm-harness = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "af7fa57c917696815937f7cdd84f48c3391e1c89" } +futures = "0.3" # Used by the asap_planner / variant_coverage bins (catalog schemas, # async SQL path, JSON output) and by the topk_ir / canonical_examples @@ -31,6 +37,8 @@ serde_yaml = "0.9" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } [dev-dependencies] +# The data generator of the harness's workloads, at the harness's pin. +aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "af7fa57c917696815937f7cdd84f48c3391e1c89" } asap-types = { path = "../types", features = ["test-support"] } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } diff --git a/crates/devtools/tests/asap_lane.rs b/crates/devtools/tests/asap_lane.rs new file mode 100644 index 000000000..fe9d189ea --- /dev/null +++ b/crates/devtools/tests/asap_lane.rs @@ -0,0 +1,510 @@ +//! The in-process ASAP lane of the engine comparison: `AsapEngine` installs a +//! plan file `asap_planner --plan-file` wrote and runs it with the Planner's +//! executor under the harness of sketch-bench (`aqpbm-harness`), as one +//! lane of `asap_run bench-external`. +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, UniformParameter}; +use aqpbm_harness::arm_run::ArmSchedule; +use aqpbm_harness::bench_external::{run_lane, CellOutcome, LaneSetup}; +use aqpbm_harness::conformance::{self, ConformanceCheck}; +use aqpbm_harness::data::{workload_chunks, DataRecord, SeriesLabels}; +use aqpbm_harness::engine::{ + ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, + QueryLanguage, QueryText, WorkloadQuery, WorkloadSpec, +}; +use aqpbm_harness::protocol::{LoadProbe, MeasurementClocks, OsCpuClock, Protocol, ProtocolError}; +use aqpbm_harness::records::ArmResult; +use aqpbm_harness::rundir::RunDir; +use asap_devtools::asap_lane::AsapEngine; +use asap_planner::{plan_installable, FrontendInput, PlanningModels, PlanningWorkload, UserInput}; + +/// A directory under the build's test temp directory, removed when dropped. +struct TempDir(PathBuf); + +impl TempDir { + fn new(test: &str) -> Self { + let path = PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(format!("asap-lane-{}-{test}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + TempDir(path) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Every core idle, so a test never waits on the machine's load. +struct IdleProbe; + +impl LoadProbe for IdleProbe { + fn core_busy_fraction(&mut self, _core: usize) -> Result { + Ok(0.0) + } + fn loadavg(&mut self) -> f64 { + 0.0 + } +} + +/// The planning workload of one exact PromQL `query` evaluated every 10 s at +/// phase 0 over gauge `metric` with `series_count` series sampled every +/// `sample_interval_ms`, arriving as `arrival` (`{"ingesting": ...}` or +/// `"at_rest"`). +fn planning_workload( + query: &str, + metric: &str, + series_count: u64, + sample_interval_ms: u64, + arrival: &str, +) -> String { + format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ + "query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {{"max_error": 0.0, "failure_probability": 0.0}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {arrival}, + "sources": {{{metric:?}: {{"series_count": {series_count}, "metric_type": "gauge", + "sample_interval": {sample_interval_ms}}}}}}}}}"# + ) +} + +/// Writes `plan_file` with `asap_planner --plan-file` for `planning`, checked +/// as `executor_check` says; whether the planner wrote it. +fn write_plan_file(planning: &str, plan_file: &Path, executor_check: &str) -> bool { + let workload_file = plan_file.with_extension("planning.json"); + std::fs::write(&workload_file, planning).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_asap_planner")) + .args(["--workload", workload_file.to_str().unwrap()]) + .args(["--run-until", "stage3", "--format", "json"]) + .args(["--executor-check", executor_check]) + .args(["--plan-file", plan_file.to_str().unwrap()]) + .output() + .unwrap(); + let written = plan_file.exists(); + assert_eq!( + output.status.success(), + written, + "{}", + String::from_utf8_lossy(&output.stderr) + ); + written +} + +/// The query of workload `w`: a 5-minute window, so a plan built at +/// ingestion time reads 30 tumbling windows of 10 s. +const SUM_QUERY: &str = "sum_over_time(m[5m])"; + +/// Workload `name` of the harness: query `sum` = `query` (PromQL) every 10 s +/// over `window_ms`, over `m` with 2 series (label `s`), one sample per +/// second from 0 for 360 s, values uniform in [0, 10), seed 1. +fn harness_workload(name: &str, query: &str, window_ms: u64) -> WorkloadSpec { + WorkloadSpec { + name: name.into(), + queries: vec![WorkloadQuery { + id: "sum".into(), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: query.into(), + }], + window_ms, + evaluation_interval_ms: 10_000, + }], + data: vec![DataRecord { + metric: "m".into(), + description: TimeSeriesDescription { + labels: vec![SeriesLabelSpec { + name: "s".into(), + cardinality: 2, + skewness: 0.0, + }], + series_num: 2, + start_ms: 0, + sample_interval_ms: 1_000, + duration_ms: 360_000, + value_distribution: DataDistribution::Uniform(UniformParameter { + lower_bound: 0.0, + upper_bound: 10.0, + seed: 0, + }), + counter: false, + seed: 1, + }, + }], + } +} + +fn write_json(path: &Path, value: &impl serde::Serialize) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, serde_json::to_string_pretty(value).unwrap()).unwrap(); +} + +/// A run directory with `workloads` and, per variant (substitutions not +/// allowed), the plan file of each workload in it. +fn run_dir(temp: &TempDir, workloads: &[&WorkloadSpec], plans: &[(&str, &str, PathBuf)]) -> RunDir { + let run = RunDir::create_at(&temp.0.join("run")).unwrap(); + for workload in workloads { + write_json( + &run + .path() + .join("workloads") + .join(format!("{}.json", workload.name)), + workload, + ); + } + for (variant, workload, plan_file) in plans { + let dir = run.path().join("plans").join(variant); + write_json( + &dir.join("variant.json"), + &serde_json::json!({"substitutions_allowed": false}), + ); + std::fs::copy(plan_file, dir.join(format!("{workload}.plan.json"))).unwrap(); + } + run +} + +/// Runs the ASAP lane over `run` with warm-up `warmup` and `runs` measured +/// runs. +fn run_asap_lane(run: &RunDir, warmup: u32, runs: u32) -> Vec { + let protocol = Protocol { + warmup_runs: warmup, + measured_runs: runs, + max_core_busy_fraction: 1.0, + }; + let clocks = MeasurementClocks::with_calling_thread(&OsCpuClock); + run_lane( + run, + &AsapEngine, + None, + LaneSetup { + protocol: &protocol, + core: allowed_core(), + probe: &mut IdleProbe, + clocks: &clocks, + args: vec!["test".into()], + }, + ) + .unwrap() +} + +/// A core this process may run on: the first of its `Cpus_allowed_list`. +fn allowed_core() -> usize { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + let list = status + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim(); + list.split([',', '-']).next().unwrap().parse().unwrap() +} + +/// `labels` without `__name__`: a PromQL `*_over_time` drops the metric name. +fn without_name(labels: &SeriesLabels) -> SeriesLabels { + let mut labels = labels.clone(); + labels.remove("__name__"); + labels +} + +/// Per series (without `__name__`), the sum of `workload`'s samples in +/// `(at_ms - window_ms, at_ms]`, computed from the regenerated samples. +fn sums_over_window( + workload: &WorkloadSpec, + window_ms: u64, + at_ms: i64, +) -> BTreeMap { + let mut sums = BTreeMap::new(); + for chunk in workload_chunks(&workload.data, 10_000).unwrap() { + for sample in chunk.unwrap().samples { + if sample.ts_ms > at_ms - window_ms as i64 && sample.ts_ms <= at_ms { + *sums.entry(without_name(&sample.labels)).or_insert(0.0) += sample.value; + } + } + } + sums +} + +fn arm_result(run: &RunDir, workload: &str, variant: &str) -> ArmResult { + let path = run + .path() + .join("results/asap") + .join(format!("{workload}__{variant}.json")); + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +/// The conformance kit's probe arm has no engine options, so this engine +/// hands `AsapEngine` the plan file of each probe workload, by name. +struct WithPlanFiles { + plan_files: BTreeMap, +} + +impl EngineUnderTest for WithPlanFiles { + fn description(&self) -> EngineDescription { + AsapEngine.description() + } + fn cpu_accounting(&self) -> CpuAccounting { + AsapEngine.cpu_accounting() + } + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let arm = ArmSpec { + engine_options: serde_json::json!({"plan_file": self.plan_files[&workload.name]}), + ..arm.clone() + }; + AsapEngine.install(workload, &arm) + } +} + +/// The harness's conformance kit passes on the ASAP lane: deterministic +/// answers, left-open windows (`count_over_time(probe[10s])` at 20 s counts +/// only the sample at 20 s), and the executor's work on the calling thread. +#[test] +fn conformance_kit_passes_on_the_asap_engine() { + let temp = TempDir::new("conformance"); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let mut plan_files = BTreeMap::new(); + for workload in conformance::probe_workloads() { + let query = &workload.queries[0].texts[0]; + assert_eq!(query.language, QueryLanguage::PromQl); + let planning: PlanningWorkload = serde_json::from_str(&planning_workload( + &query.text, + "probe", + 1, + 10_000, + r#"{"ingesting": {"rows_per_second": 0.1}}"#, + )) + .unwrap(); + let (plan, _) = runtime + .block_on(plan_installable(UserInput::new( + &planning, + FrontendInput::Promql, + PlanningModels::builtin(), + ))) + .unwrap(); + let path = temp.0.join(format!("{}.plan.json", workload.name)); + std::fs::write(&path, serde_json::to_string(&plan).unwrap()).unwrap(); + plan_files.insert(workload.name.clone(), path); + } + let mut engine = WithPlanFiles { plan_files }; + let report = conformance::check( + &mut engine, + &MeasurementClocks::with_calling_thread(&OsCpuClock), + ) + .unwrap_or_else(|failures| panic!("{failures:#?}")); + for check in [ + ConformanceCheck::DeterministicAnswers, + ConformanceCheck::LeftOpenWindow, + ConformanceCheck::WorkOnCallingThread, + ] { + assert!(report.passed.contains(&check), "{report:?}"); + } +} + +/// `sum_over_time(m[5m])` every 10 s over 2 series, planned twice: with +/// samples arriving (a chain of 10 s tumbling windows built at ingestion +/// time) and at rest (raw samples scanned at query time). The ASAP lane +/// runs both plans (one warm-up and two measured runs, so three installs +/// interleaved) and answers every scheduled evaluation exactly: per series, +/// the sum of its samples in `(T - 5 min, T]`. +#[test] +fn the_asap_lane_runs_a_planned_sum_over_time_and_answers_exactly() { + let temp = TempDir::new("sum"); + let workload = harness_workload("w", SUM_QUERY, 300_000); + let ingesting = temp.0.join("ingesting.plan.json"); + let at_rest = temp.0.join("at-rest.plan.json"); + for (plan_file, arrival) in [ + (&ingesting, r#"{"ingesting": {"rows_per_second": 1000.0}}"#), + (&at_rest, r#""at_rest""#), + ] { + let planning = planning_workload(SUM_QUERY, "m", 1_000, 1_000, arrival); + assert!(write_plan_file(&planning, plan_file, "compile")); + } + let run = run_dir( + &temp, + &[&workload], + &[("ingesting", "w", ingesting), ("at-rest", "w", at_rest)], + ); + let cells = run_asap_lane(&run, 1, 2); + assert_eq!(cells.len(), 2, "{cells:?}"); + assert!( + cells.iter().all(|cell| cell.outcome == CellOutcome::Ran), + "{cells:?}" + ); + + let schedule = ArmSchedule::for_workload(&workload).unwrap(); + for variant in ["ingesting", "at-rest"] { + let result = arm_result(&run, "w", variant); + assert!(result.plan.substitutions.is_empty()); + assert!(result.plan.sharing_lost.is_empty()); + assert!(!result.plan.node_kinds.is_empty()); + let times: Vec = result.answers.iter().map(|(at_ms, _)| *at_ms).collect(); + assert_eq!(times, schedule.evaluate_at_ms, "{variant}"); + for (at_ms, answers) in &result.answers { + let expected = sums_over_window(&workload, 300_000, *at_ms); + assert_eq!(expected.len(), 2); + let answered = &answers.0["sum"]; + assert_eq!( + answered.keys().collect::>(), + expected.keys().collect::>(), + "{variant} at {at_ms}" + ); + for (series, value) in &expected { + let got = answered[series]; + assert!( + (got - value).abs() <= 1e-9 * value.abs(), + "{variant} at {at_ms} {series:?}: {got} vs {value}" + ); + } + } + // The plan's shape, a precondition: if the planner changes it, pick + // workloads that keep one ingestion-time and one query-time plan. + match variant { + "ingesting" => { + assert!(!result.plan.stored_outputs.is_empty(), "{:?}", result.plan); + assert!(result.footprint.view_bytes > 0, "{:?}", result.footprint); + } + _ => { + assert!(result.plan.stored_outputs.is_empty(), "{:?}", result.plan); + assert!( + result.footprint.raw_retained_bytes > 0, + "{:?}", + result.footprint + ); + } + } + } +} + +/// `last_over_time(m[10s])`: Stages 1-3 plan it, but the Planner's executor +/// has no `last_over_time` kernel. The plan file written for a runtime other +/// than the executor (`--executor-check skip`) is a "not executable" cell +/// on the ASAP lane, so the other lanes still run that plan. +#[test] +fn a_plan_the_executor_cannot_compile_is_a_not_executable_cell() { + const QUERY: &str = "last_over_time(m[10s])"; + let temp = TempDir::new("last"); + let planning = planning_workload(QUERY, "m", 2, 1_000, r#""at_rest""#); + let plan_file = temp.0.join("last.plan.json"); + // Precondition: an executor-only gap. Once the executor runs + // `last_over_time`, pick another query it plans but cannot compile. + assert!(!write_plan_file(&planning, &plan_file, "compile")); + assert!(write_plan_file(&planning, &plan_file, "skip")); + let workload = harness_workload("w", QUERY, 10_000); + let run = run_dir(&temp, &[&workload], &[("exact", "w", plan_file)]); + let cells = run_asap_lane(&run, 0, 1); + let [cell] = cells.as_slice() else { + panic!("{cells:?}") + }; + match &cell.outcome { + CellOutcome::NotRun { error } => { + assert!(error.starts_with("not executable: "), "{error}") + } + other => panic!("{other:?}"), + } +} + +/// `install`'s refusals: no plan file in the engine options, a missing file, +/// a file that is not a plan, and a plan for other queries (query `i` is +/// answered by the plan's root `i`, matched by text in any of the query's +/// languages). +#[test] +fn a_plan_for_other_queries_is_refused_naming_the_query() { + let temp = TempDir::new("refusals"); + let planning = planning_workload(SUM_QUERY, "m", 2, 1_000, r#""at_rest""#); + let plan_file = temp.0.join("sum.plan.json"); + assert!(write_plan_file(&planning, &plan_file, "compile")); + let arm = |options: serde_json::Value| ArmSpec { + name: "exact".into(), + raw_data_retained: true, + engine_options: options, + }; + let with_plan = arm(serde_json::json!({ "plan_file": plan_file })); + let install_error = |workload: &WorkloadSpec, arm: &ArmSpec| match AsapEngine.install(workload, arm) { + Ok(_) => panic!("installed"), + Err(EngineError(message)) => message, + }; + let workload = harness_workload("w", SUM_QUERY, 300_000); + + let error = install_error(&workload, &arm(serde_json::Value::Null)); + assert!(error.contains("plan_file"), "{error}"); + let missing = temp.0.join("missing.plan.json"); + let error = install_error(&workload, &arm(serde_json::json!({ "plan_file": missing }))); + assert!(error.contains("missing.plan.json"), "{error}"); + let not_a_plan = temp.0.join("not-a-plan.json"); + std::fs::write(¬_a_plan, "{}").unwrap(); + let error = install_error(&workload, &arm(serde_json::json!({ "plan_file": not_a_plan }))); + assert!(error.contains("not-a-plan.json"), "{error}"); + + // Another query text: the error names the query and both texts. + let other = harness_workload("w", "count_over_time(m[5m])", 300_000); + let error = install_error(&other, &with_plan); + for part in ["query 0", "sum", SUM_QUERY, "count_over_time(m[5m])"] { + assert!(error.contains(part), "{part}: {error}"); + } + // Two queries against a plan with one root. + let mut two = workload.clone(); + two.queries.push(WorkloadQuery { + id: "second".into(), + ..two.queries[0].clone() + }); + let error = install_error(&two, &with_plan); + assert!(error.contains("1 query text") && error.contains("2 queries"), "{error}"); + + // The plan's text as the query's second text (after a SQL text) matches. + let mut sql_first = workload.clone(); + sql_first.queries[0].texts.insert( + 0, + QueryText { + language: QueryLanguage::Sql, + text: "SELECT s, sum(value) FROM m GROUP BY s".into(), + }, + ); + assert!(AsapEngine.install(&sql_first, &with_plan).is_ok()); +} + +/// `asap_run bench-external --lane asap` runs the lane over a run directory +/// and writes each cell; an unknown lane is a usage error naming the known +/// one. +#[test] +fn asap_run_runs_the_asap_lane_over_a_run_dir() { + let temp = TempDir::new("asap-run"); + let workload = harness_workload("w", SUM_QUERY, 300_000); + let plan_file = temp.0.join("sum.plan.json"); + let planning = planning_workload(SUM_QUERY, "m", 2, 1_000, r#""at_rest""#); + assert!(write_plan_file(&planning, &plan_file, "compile")); + let run = run_dir(&temp, &[&workload], &[("exact", "w", plan_file)]); + let asap_run = |lane: &str| { + Command::new(env!("CARGO_BIN_EXE_asap_run")) + .args(["bench-external", "--lane", lane]) + .args(["--run-dir", run.path().to_str().unwrap()]) + .args(["--core", &allowed_core().to_string()]) + .args(["--warmup", "0", "--runs", "1", "--max-core-busy", "1.0"]) + .output() + .unwrap() + }; + let output = asap_run("asap"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let cell: aqpbm_harness::bench_external::LaneCell = serde_json::from_str( + &std::fs::read_to_string(run.path().join("results/asap/w__exact.cell.json")).unwrap(), + ) + .unwrap(); + assert_eq!(cell.outcome, CellOutcome::Ran); + + let output = asap_run("fake"); + assert_eq!(output.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&output.stderr).contains("asap")); +} diff --git a/crates/devtools/tests/main.rs b/crates/devtools/tests/main.rs index ed12546d0..82c45c766 100644 --- a/crates/devtools/tests/main.rs +++ b/crates/devtools/tests/main.rs @@ -2,6 +2,7 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-devtools --test devtools ::`. +mod asap_lane; mod asap_planner; mod asap_planner_acceptance; mod cross_language; diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 192d63371..638e8cb7d 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -994,3 +994,102 @@ fn a_materialized_view_is_requested_for_the_window_it_summarizes() { ); } } + +/// Rows `(ts, value)` with one row at each of `times`, in `schema`. +fn rows_at(schema: &SchemaRef, times: &[i64]) -> Batch { + Batch::try_new( + schema.clone(), + times + .iter() + .map(|&ts| vec![Value::Timestamp(ts), Value::Float64(ts as f64)]) + .collect(), + ) + .unwrap() +} + +/// The times of the rows `store` returns for `source` read over `range`. +fn times_read( + store: &InMemoryDataStore, + source: &Source, + schema: &SchemaRef, + range: Option<(i64, i64)>, +) -> Vec { + let request = RawDataRequest { + time_range_ms: range, + schema, + predicates: &[], + }; + let batches: Vec = block_on( + store + .read_raw_data(source, &request, context()) + .unwrap() + .map(Result::unwrap) + .collect(), + ); + batches + .iter() + .flat_map(Batch::to_rows) + .map(|row| match row[0] { + Value::Timestamp(ts) => ts, + ref other => panic!("{other:?}"), + }) + .collect() +} + +/// A runtime appends a time series' raw rows as they arrive, each batch with +/// the `(start, end]` it covers: a read returns the batches overlapping the +/// requested range (the executor re-applies the range), and appending drops +/// the batches ending `lookback_ms` or more before the newest end. E.g. +/// lookback 15 ms at end 30 drops `(0, 10]` and keeps `(10, 20]`, which +/// holds rows a window `(15, 30]` reads. +#[test] +fn appended_raw_data_is_read_by_range_and_dropped_after_its_lookback() { + let schema = Arc::new(planner_types::ir::schema::Schema { + unique_keys: vec![], + closed: false, + fields: vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + time_index: Some(0), + }); + let source = Source::TimeSeries { metric: "m".into() }; + let mut store = InMemoryDataStore::default() + .with_appended_raw_data(source.clone(), schema.clone(), Some(15)) + .unwrap(); + assert_eq!(store.raw_data_bytes(), 0); + for (start, end) in [(0, 10), (10, 20)] { + store + .append_raw_data(&source, (start, end), vec![rows_at(&schema, &[end])]) + .unwrap(); + } + assert_eq!(times_read(&store, &source, &schema, Some((5, 20))), [10, 20]); + assert_eq!(times_read(&store, &source, &schema, Some((10, 20))), [20]); + store + .append_raw_data(&source, (20, 30), vec![rows_at(&schema, &[25, 30])]) + .unwrap(); + assert_eq!(times_read(&store, &source, &schema, None), [20, 25, 30]); + assert_eq!(times_read(&store, &source, &schema, Some((15, 30))), [20, 25, 30]); + assert_eq!(times_read(&store, &source, &schema, Some((20, 30))), [25, 30]); + let held = rows_at(&schema, &[20]).memory_bytes() + rows_at(&schema, &[25, 30]).memory_bytes(); + assert_eq!(store.raw_data_bytes(), held); + assert_eq!(store.materialized_view_bytes(), 0); + + // Append-only, to a source bound for appending. + assert!(store + .append_raw_data(&source, (25, 40), vec![rows_at(&schema, &[40])]) + .is_err()); + let other = Source::TimeSeries { metric: "other".into() }; + assert!(store + .append_raw_data(&other, (30, 40), vec![rows_at(&schema, &[40])]) + .is_err()); + let fixed = Source::TimeSeries { metric: "fixed".into() }; + let mut store = store + .with_raw_data(fixed.clone(), schema.clone(), vec![rows_at(&schema, &[1])]) + .unwrap(); + assert!(store + .append_raw_data(&fixed, (30, 40), vec![rows_at(&schema, &[40])]) + .is_err()); + // Rows bound whole are returned whatever the range. + assert_eq!(times_read(&store, &fixed, &schema, Some((20, 30))), [1]); +} From ed32ada328c4c4aab5ed2bd6b9628980072a155d Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:39:47 +0000 Subject: [PATCH 2/7] feat: the in-process ASAP lane of the engine comparison AsapEngine (asap_devtools::asap_lane) installs a plan file written by asap_planner --plan-file and runs it with the Planner's executor as an engine under test of the sketch-bench harness; asap_run bench-external --lane asap runs it over a run directory. A plan the executor cannot compile is an install error starting 'not executable:'. The lane is a deployment runtime over the executor's public API: the executor's InMemoryDataStore gains raw tables bound for appending, read by the batches overlapping the requested range and dropped after their lookback, and byte accessors; InstallablePlan exposes the compiled query run and ingestion refreshes its compile check already built; the RawDataRetention accessors are public, since a runtime reads them. --- .../devtools/src/asap_lane/installed_plan.rs | 349 ++++++++++++++++++ crates/devtools/src/asap_lane/mod.rs | 135 +++++++ .../src/asap_lane/time_series_batches.rs | 218 +++++++++++ crates/devtools/src/bin/asap_run.rs | 128 +++++++ crates/devtools/src/lib.rs | 5 +- crates/devtools/tests/asap_lane.rs | 32 +- crates/executor/src/sources/memory.rs | 198 ++++++++-- crates/planner/src/installable/mod.rs | 65 +++- .../types/src/plan/materialized_view/mod.rs | 4 + 9 files changed, 1081 insertions(+), 53 deletions(-) create mode 100644 crates/devtools/src/asap_lane/installed_plan.rs create mode 100644 crates/devtools/src/asap_lane/mod.rs create mode 100644 crates/devtools/src/asap_lane/time_series_batches.rs create mode 100644 crates/devtools/src/bin/asap_run.rs diff --git a/crates/devtools/src/asap_lane/installed_plan.rs b/crates/devtools/src/asap_lane/installed_plan.rs new file mode 100644 index 000000000..60ccb259c --- /dev/null +++ b/crates/devtools/src/asap_lane/installed_plan.rs @@ -0,0 +1,349 @@ +//! One installed plan of the ASAP lane: its compiled runs and its store. + +use std::sync::Arc; + +use aqpbm_harness::data::Sample; +use aqpbm_harness::engine::{ + Answers, EngineError, Footprint, InstalledArm, PlanSummary, SampleChunk, StepReport, + WorkloadSpec, +}; +use asap_executor::physical_planner::CompiledPhysicalDAG; +use asap_executor::runtime::{Limits, RunContext, Scope}; +use asap_executor::sources::{DataStore, InMemoryDataStore}; +use asap_executor::values::Batch; +use asap_planner::{InstallablePlan, MaterializedView, MaterializedViewRefresh, RefreshWindow}; +use futures::executor::block_on; +use futures::StreamExt; + +use super::time_series_batches::{ + answers_of, time_series_tables, SeriesIdentities, TimeSeriesTable, +}; + +/// The memory one run may reserve: no limit in practice, so the lane +/// measures the plan, not a limit of this runtime. +const RUN_MEMORY_LIMIT_BYTES: usize = 1 << 40; + +/// An installed plan: the time series it reads, its ingestion refresh +/// groups and query run compiled once, and the store holding its raw +/// samples and stored views. +pub(super) struct InstalledPlan { + plan: InstallablePlan, + /// The workload query each root answers, in root order. + query_ids: Vec, + tables: Vec, + identities: SeriesIdentities, + refreshes: Vec, + query_run: CompiledPhysicalDAG, + /// Shared with each run while it runs, held alone between steps. + store: Arc, + /// The end of the samples ingested so far: the data start before the + /// first chunk. + ingested_until_ms: i64, +} + +/// One ingestion refresh group: refreshed by one ingestion run per window +/// `(e - width, e]` with `e + phase ≡ 0 (mod width)`, whose roots are its +/// views. +struct GroupRefresh { + window: RefreshWindow, + views: Vec, + compiled: CompiledPhysicalDAG, +} + +/// Bad input: the lane cannot run this plan, though the executor may. +fn unsupported(reason: impl std::fmt::Display) -> EngineError { + EngineError(format!("unsupported by the ASAP lane: {reason}")) +} + +/// The executor cannot compile this plan. +fn not_executable(reason: impl std::fmt::Display) -> EngineError { + EngineError(format!("not executable: {reason}")) +} + +fn run_failed(step: &str, reason: impl std::fmt::Display) -> EngineError { + EngineError(format!("{step}: {reason}")) +} + +impl InstalledPlan { + /// Compiles `plan`'s query run and each ingestion refresh group's + /// refresh, and binds an empty store: each time series the plan reads, + /// kept as far back as its query-time `Scan`s read and the refreshes + /// need. `query_ids` names the query of each root. + pub(super) fn install( + plan: InstallablePlan, + query_ids: Vec, + workload: &WorkloadSpec, + ) -> Result { + let query_run = plan.compile_query_run().map_err(not_executable)?; + let groups = plan.ingestion_refresh_groups().map_err(not_executable)?; + let mut refreshes = Vec::with_capacity(groups.len()); + for group in &groups { + let compiled = plan + .compile_ingestion_refresh(group) + .map_err(not_executable)?; + let first = group.views[0].node_id; + let window = group + .refresh_window + .ok_or_else(|| unsupported(format!("view {first} has no bounded window")))?; + refreshes.push(GroupRefresh { + window, + views: group.views.clone(), + compiled, + }); + } + let views = plan.materialized_views().map_err(not_executable)?; + if let Some(view) = views + .iter() + .find(|view| view.refresh() == MaterializedViewRefresh::OnFirstRead) + { + return Err(unsupported(format!( + "view {} is refreshed on first read", + view.node_id + ))); + } + let tables = time_series_tables(&plan.dag).map_err(unsupported)?; + let retention = plan.raw_data_retention().map_err(not_executable)?; + // A refresh reads the window ending at a chunk's end or before it, + // so the raw samples it reads reach back one window and one chunk. + let chunk_ms = workload + .queries + .iter() + .map(|query| query.evaluation_interval_ms) + .max() + .unwrap_or(0); + let refresh_lookback_ms = refreshes + .iter() + .map(|refresh| refresh.window.width_ms + chunk_ms) + .max() + .unwrap_or(0); + let mut store = InMemoryDataStore::default(); + for table in &tables { + let query_lookback_ms = retention + .iter() + .filter(|kept| kept.source() == &table.source) + .map(|kept| kept.lookback_ms()) + .try_fold(0, |longest, lookback| Some(longest.max(lookback?))); + let lookback_ms = query_lookback_ms.map(|ms| ms.max(refresh_lookback_ms)); + store = store + .with_appended_raw_data(table.source.clone(), table.schema.clone(), lookback_ms) + .map_err(unsupported)?; + } + let ingested_until_ms = workload + .data + .iter() + .map(|record| record.description.start_ms) + .min() + .unwrap_or(0); + Ok(InstalledPlan { + plan, + query_ids, + tables, + identities: SeriesIdentities::default(), + refreshes, + query_run, + store: Arc::new(store), + ingested_until_ms, + }) + } + + /// The store, between runs: no run holds it then. + fn store_mut(&mut self) -> &mut InMemoryDataStore { + Arc::get_mut(&mut self.store).expect("no run holds the store between steps") + } + + /// Refreshes `refresh` for the window ending at `end_ms`: one ingestion + /// run whose roots are its views, each view's output appended. + fn refresh_window(&mut self, refresh: usize, end_ms: i64) -> Result<(), EngineError> { + let GroupRefresh { + window, + views, + compiled, + } = &self.refreshes[refresh]; + let start_ms = end_ms - window.width_ms as i64; + let scope = Scope::Ingestion { + window_start_ms: start_ms, + window_end_ms: end_ms, + }; + let outputs = run_ingestion(compiled, &self.store, scope) + .map_err(|error| run_failed(&format!("refresh of ({start_ms}, {end_ms}]"), error))?; + let views = views.clone(); + let store = self.store_mut(); + for (view, batches) in views.iter().zip(outputs) { + store + .append_materialized_view(view, (start_ms, end_ms), batches) + .map_err(|error| run_failed("storing a view", error))?; + } + Ok(()) + } +} + +/// `compiled` run in ingestion `scope` over `store`: each root's batches. +fn run_ingestion( + compiled: &CompiledPhysicalDAG, + store: &Arc, + scope: Scope, +) -> Result>, asap_executor::Error> { + let instance = compiled.instantiate(Arc::clone(store) as Arc)?; + let context = RunContext::new(scope, run_limits())?; + let outputs = block_on(instance.execute_ingestion(compiled.roots(), context)?)?; + Ok(outputs + .into_iter() + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .collect()) +} + +/// `compiled` run at `evaluation_time_ms` over `store`: each root's batches. +fn run_query( + compiled: &CompiledPhysicalDAG, + store: &Arc, + evaluation_time_ms: i64, +) -> Result>, asap_executor::Error> { + let instance = compiled.instantiate(Arc::clone(store) as Arc)?; + let context = RunContext::new(Scope::Query { evaluation_time_ms }, run_limits())?; + let outputs = instance.execute(compiled.roots(), context)?; + block_on(async { + let mut per_root = Vec::with_capacity(outputs.len()); + for mut output in outputs { + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + batches.push((*batch?).clone()); + } + per_root.push(batches); + } + Ok(per_root) + }) +} + +fn run_limits() -> Limits { + Limits { + max_bytes: RUN_MEMORY_LIMIT_BYTES, + } +} + +/// The ends `e` in `(after_ms, until_ms]` of `window`'s windows: `e + +/// phase ≡ 0 (mod width)`, since a window ending `phase` before an +/// evaluation on the width's grid ends there. Example: width 10 s, phase 0, +/// `(0, 30 s]`: 10, 20 and 30 s. +fn window_ends(window: RefreshWindow, after_ms: i64, until_ms: i64) -> impl Iterator { + let width = window.width_ms as i64; + let phase = window.phase_ms as i64; + let first = after_ms + (-phase - after_ms).rem_euclid(width); + let first = if first == after_ms { + first + width + } else { + first + }; + (0..) + .map(move |k| first + k * width) + .take_while(move |&end| end <= until_ms) +} + +impl InstalledArm for InstalledPlan { + /// One line per DAG node (id, operator, timing, lifetime, retention), + /// its operators, and its stored views; the lane runs the plan as + /// given, so no substitutions and no sharing lost. + fn plan_summary(&self) -> PlanSummary { + let dag = &self.plan.dag; + let text = dag + .nodes + .iter() + .map(|node| { + format!( + "n{} {} {:?} {:?} retention_ms {:?}", + node.id, + node.payload.kind_name(), + node.output_state.timing, + node.lifetime, + node.retention_ms + ) + }) + .chain(std::iter::once(format!("roots {:?}", dag.roots))) + .collect::>() + .join("\n"); + let stored_outputs = self + .plan + .materialized_views() + .unwrap_or_default() + .iter() + .map(|view| { + format!( + "n{} refreshed {:?}, kept {} ms", + view.node_id, + view.refresh(), + view.retention_ms() + ) + }) + .collect(); + PlanSummary { + text, + node_kinds: dag + .nodes + .iter() + .map(|node| node.payload.kind_name().to_string()) + .collect(), + stored_outputs, + substitutions: Vec::new(), + sharing_lost: Vec::new(), + } + } + + /// Appends the chunk's samples of each time series the plan reads as + /// its rows for `(previous end, chunk end]`, then refreshes each group + /// for every window ending in that range. + fn ingest(&mut self, chunk: &SampleChunk) -> Result { + let (after_ms, until_ms) = (self.ingested_until_ms, chunk.end_ms); + let mut batches = Vec::with_capacity(self.tables.len()); + for table in &self.tables { + let samples: Vec<&Sample> = chunk + .samples + .iter() + .filter(|sample| sample.labels.get("__name__") == Some(&table.metric)) + .collect(); + let batch = table + .batch(&samples, &mut self.identities) + .map_err(|error| run_failed("raw samples", error))?; + batches.push(batch); + } + let sources: Vec<_> = self + .tables + .iter() + .map(|table| table.source.clone()) + .collect(); + let store = self.store_mut(); + for (source, batch) in sources.iter().zip(batches) { + store + .append_raw_data(source, (after_ms, until_ms), vec![batch]) + .map_err(|error| run_failed("raw samples", error))?; + } + for refresh in 0..self.refreshes.len() { + let ends: Vec = + window_ends(self.refreshes[refresh].window, after_ms, until_ms).collect(); + for end_ms in ends { + self.refresh_window(refresh, end_ms)?; + } + } + self.ingested_until_ms = until_ms; + Ok(StepReport::default()) + } + + /// One query run at `at_ms`: root `i`'s rows answer query `i`. + fn evaluate(&mut self, at_ms: i64) -> Result<(Answers, StepReport), EngineError> { + let per_root = run_query(&self.query_run, &self.store, at_ms) + .map_err(|error| run_failed(&format!("query run at {at_ms}"), error))?; + let mut answers = Answers::default(); + for (query_id, batches) in self.query_ids.iter().zip(&per_root) { + let series = answers_of(batches).map_err(|error| run_failed(query_id, error))?; + answers.0.insert(query_id.clone(), series); + } + Ok((answers, StepReport::default())) + } + + /// The bytes of the stored views and of the raw samples held. + fn footprint(&self) -> Result { + Ok(Footprint { + view_bytes: self.store.materialized_view_bytes() as u64, + raw_retained_bytes: self.store.raw_data_bytes() as u64, + peak_bytes: None, + }) + } +} diff --git a/crates/devtools/src/asap_lane/mod.rs b/crates/devtools/src/asap_lane/mod.rs new file mode 100644 index 000000000..31d6a9ba0 --- /dev/null +++ b/crates/devtools/src/asap_lane/mod.rs @@ -0,0 +1,135 @@ +//! The in-process ASAP lane of the engine comparison: the Planner's +//! executor as an engine under test of the sketch-bench harness +//! (`aqpbm-harness`). +//! +//! [`AsapEngine`] installs a plan file `asap_planner --plan-file` wrote (an +//! [`InstallablePlan`]) as given, so this lane and every other lane of a +//! run execute the same DAG; `asap_run bench-external --lane asap` runs it +//! over a run directory. The lane is a deployment runtime over the +//! executor's public API: it keeps the raw samples and the stored views in +//! one [`InMemoryDataStore`](asap_executor::sources::InMemoryDataStore), refreshes each ingestion refresh group in one +//! ingestion run per window as samples arrive, and answers each evaluation +//! with one query run, all on the calling thread. +//! +//! Example: a plan for `sum_over_time(m[5m])` every 10 s built at ingestion +//! time is a chain of 10 s tumbling windows: each ingested chunk of 10 s +//! refreshes the window ending at its end, and the query run at `T` merges +//! the 30 windows ending in `(T - 5 min, T]`. + +mod installed_plan; +mod time_series_batches; + +use std::path::PathBuf; + +use aqpbm_harness::engine::{ + ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, + WorkloadSpec, +}; +use asap_planner::InstallablePlan; + +use installed_plan::InstalledPlan; + +/// The Planner's executor as an engine under test: installs the plan file +/// named by `arm.engine_options["plan_file"]`. +/// +/// Its version is the commit in `ASAP_PLANNER_COMMIT` at build time +/// (`unknown` without it), dirty when `ASAP_PLANNER_DIRTY` is `1`; e.g. +/// `ASAP_PLANNER_COMMIT=$(git rev-parse HEAD) cargo build --release --bin asap_run`. +#[derive(Debug, Clone, Copy, Default)] +pub struct AsapEngine; + +impl EngineUnderTest for AsapEngine { + fn description(&self) -> EngineDescription { + EngineDescription { + id: "asap".into(), + version: option_env!("ASAP_PLANNER_COMMIT") + .unwrap_or("unknown") + .into(), + version_dirty: option_env!("ASAP_PLANNER_DIRTY") == Some("1"), + build_profile: if cfg!(debug_assertions) { + "debug" + } else { + "release" + } + .into(), + } + } + + /// Every ingestion and query run runs on the thread that calls + /// `ingest` or `evaluate` (`futures::executor::block_on`). + fn cpu_accounting(&self) -> CpuAccounting { + CpuAccounting::CallingThread + } + + /// Reads and validates the plan file, matches its query texts to + /// `workload`'s queries, and compiles it with the executor. A plan the + /// executor cannot compile is an error starting `not executable:`, so + /// the cell reads "not run: not executable: ..." while the other lanes + /// still run that plan. + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let plan_file = plan_file_of(arm)?; + let plan = read_plan_file(&plan_file)?; + let query_ids = query_ids_of(&plan, workload) + .map_err(|error| EngineError(format!("plan file {}: {error}", plan_file.display())))?; + Ok(Box::new(InstalledPlan::install(plan, query_ids, workload)?)) + } +} + +/// `arm.engine_options["plan_file"]`. +fn plan_file_of(arm: &ArmSpec) -> Result { + arm.engine_options + .get("plan_file") + .and_then(serde_json::Value::as_str) + .map(PathBuf::from) + .ok_or_else(|| { + EngineError(format!( + "arm {}: engine options {} name no plan_file", + arm.name, arm.engine_options + )) + }) +} + +/// The plan file at `path`, read and validated. +fn read_plan_file(path: &std::path::Path) -> Result { + let error = |error: &dyn std::fmt::Display| { + EngineError(format!("plan file {}: {error}", path.display())) + }; + let text = std::fs::read_to_string(path).map_err(|e| error(&e))?; + let plan = InstallablePlan::from_json(&text).map_err(|e| error(&e))?; + plan.validate().map_err(|e| error(&e))?; + Ok(plan) +} + +/// The id of the query each root of `plan` answers: root `i` answers query +/// text `i`, which must be one of the texts (in any language) of +/// `workload`'s query `i`. Example: a plan for `sum_over_time(m[5m])` and a +/// workload whose query `sum` reads `sum_over_time(m[5m])` give `["sum"]`. +fn query_ids_of(plan: &InstallablePlan, workload: &WorkloadSpec) -> Result, String> { + if plan.query_texts.len() != workload.queries.len() { + return Err(format!( + "{} query text(s) for workload {}'s {} queries", + plan.query_texts.len(), + workload.name, + workload.queries.len() + )); + } + plan.query_texts + .iter() + .zip(&workload.queries) + .enumerate() + .map(|(index, (planned, query))| { + if query.texts.iter().any(|text| &text.text == planned) { + return Ok(query.id.clone()); + } + let texts: Vec<&str> = query.texts.iter().map(|text| text.text.as_str()).collect(); + Err(format!( + "query {index} ({}) is planned as {planned:?}, but the workload's texts are {texts:?}", + query.id + )) + }) + .collect() +} diff --git a/crates/devtools/src/asap_lane/time_series_batches.rs b/crates/devtools/src/asap_lane/time_series_batches.rs new file mode 100644 index 000000000..ccf10e8f3 --- /dev/null +++ b/crates/devtools/src/asap_lane/time_series_batches.rs @@ -0,0 +1,218 @@ +//! Samples in, answers out: the raw table of each time series the plan's +//! `Scan`s read, built column by column from the harness's samples, and the +//! series and value of each answer row. +//! +//! A time series is the table `(ts, value, $promql_series_identity, +//! *labels)` over its samples (the columns its `Scan`s read): the series +//! identity is the canonical JSON of every label, `__name__` included, +//! e.g. `{"__name__":"m","s":"s_0"}`. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use aqpbm_harness::data::{Sample, SeriesLabels}; +use asap_executor::arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array, +}; +use asap_executor::arrow::datatypes::UInt32Type; +use asap_executor::values::{Batch, BatchColumn, SchemaRef, Value}; +use asap_planner::PhysicalASAPDAG; +use asap_types::ir::operator::Source; +use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; +use asap_types::ir::NonASAPOp; +use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; + +/// One time series' raw table: what each of its columns holds. +pub(super) struct TimeSeriesTable { + /// `Source::TimeSeries` of `metric`. + pub(super) source: Source, + pub(super) metric: String, + /// The union of its `Scan`s' columns, by name. + pub(super) schema: SchemaRef, + columns: Vec, +} + +/// What one column of a time series' raw table holds. +enum TimeSeriesColumn { + /// The sample's time, ms. + Time, + /// The sample's value. + Value, + /// The canonical JSON of the series' labels. + SeriesIdentity, + /// The value of this label; empty when the series lacks it. + Label(String), +} + +/// The raw table of each time series `dag`'s `Scan`s read, in metric order. +/// `Err` for a source the lane cannot feed: a SQL table, or a column that +/// is neither the time, the value, the series identity nor a text label. +pub(super) fn time_series_tables(dag: &PhysicalASAPDAG) -> Result, String> { + let mut schemas: BTreeMap<&str, Schema> = BTreeMap::new(); + for node in &dag.nodes { + let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, schema, .. }) = + &node.payload + else { + continue; + }; + let Source::TimeSeries { metric } = source else { + return Err(format!( + "node {} scans {source:?}, not a time series", + node.id + )); + }; + let known = schemas.entry(metric).or_insert_with(|| schema.clone()); + for field in &schema.fields { + if !known.fields.iter().any(|known| known.name == field.name) { + known.fields.push(field.clone()); + } + } + } + schemas + .into_iter() + .map(|(metric, schema)| { + let columns = schema + .fields + .iter() + .enumerate() + .map(|(index, field)| match (&field.dtype, field.name.as_str()) { + _ if Some(index) == schema.time_index => Ok(TimeSeriesColumn::Time), + (FieldDataType::Plain(DataType::Float64), "value") => { + Ok(TimeSeriesColumn::Value) + } + (FieldDataType::Plain(DataType::Utf8), PROMQL_SERIES_IDENTITY) => { + Ok(TimeSeriesColumn::SeriesIdentity) + } + (FieldDataType::Plain(DataType::Utf8), label) => { + Ok(TimeSeriesColumn::Label(label.to_string())) + } + (dtype, name) => Err(format!( + "time series {metric}: column {name} of type {dtype:?} is not a label" + )), + }) + .collect::>()?; + Ok(TimeSeriesTable { + source: Source::TimeSeries { + metric: metric.to_string(), + }, + metric: metric.to_string(), + schema: Arc::new(schema), + columns, + }) + }) + .collect() +} + +/// Each series' identity text, computed once per series: the harness shares +/// one label map among a series' samples, so the map's address names the +/// series. The map is kept alive here, so its address is never reused by +/// another series. +#[derive(Default)] +pub(super) struct SeriesIdentities { + by_address: HashMap<*const SeriesLabels, (Arc, Arc)>, +} + +impl SeriesIdentities { + fn identity(&mut self, labels: &Arc) -> Arc { + let (_, identity) = self + .by_address + .entry(Arc::as_ptr(labels)) + .or_insert_with(|| { + let json = serde_json::to_string(labels.as_ref()).expect("labels are JSON"); + (Arc::clone(labels), json.into()) + }); + Arc::clone(identity) + } +} + +impl TimeSeriesTable { + /// The rows of `samples` (all of this table's metric) in the table's + /// schema. Every text column is dictionary-encoded over the chunk's + /// series: one key per row, one text per series. + pub(super) fn batch( + &self, + samples: &[&Sample], + identities: &mut SeriesIdentities, + ) -> Result { + // Each row's series as an index into this chunk's series. + let mut series: Vec<&Arc> = Vec::new(); + let mut index_of: HashMap<*const SeriesLabels, u32> = HashMap::new(); + let keys: UInt32Array = samples + .iter() + .map(|sample| { + *index_of + .entry(Arc::as_ptr(&sample.labels)) + .or_insert_with(|| { + series.push(&sample.labels); + (series.len() - 1) as u32 + }) + }) + .collect(); + let text_column = |texts: StringArray| -> Result { + let column = DictionaryArray::::try_new(keys.clone(), Arc::new(texts)) + .map_err(|error| asap_executor::Error::Invalid(error.to_string()))?; + Ok(BatchColumn::Plain(Arc::new(column) as ArrayRef)) + }; + let columns = self + .columns + .iter() + .map(|column| match column { + TimeSeriesColumn::Time => Ok(BatchColumn::Plain(Arc::new( + TimestampMillisecondArray::from_iter_values( + samples.iter().map(|sample| sample.ts_ms), + ), + ))), + TimeSeriesColumn::Value => Ok(BatchColumn::Plain(Arc::new( + Float64Array::from_iter_values(samples.iter().map(|sample| sample.value)), + ))), + TimeSeriesColumn::SeriesIdentity => { + let texts: Vec> = series + .iter() + .map(|labels| identities.identity(labels)) + .collect(); + text_column(texts.iter().map(|text| Some(text.as_ref())).collect()) + } + TimeSeriesColumn::Label(label) => text_column( + series + .iter() + .map(|labels| Some(labels.get(label).map_or("", String::as_str))) + .collect(), + ), + }) + .collect::, _>>()?; + Batch::try_from_columns(self.schema.clone(), columns) + } +} + +/// Each answer row of `batches` (one query root's output) as its series +/// labels and value: the series identity's labels, when the output keeps +/// one, and every other non-empty text column (a label the query grouped +/// by); the value is the row's first Float64 column. Example: a row +/// `(ts, 12.5, {"s":"s_0"})` of `sum_over_time(m[5m])` is `{s="s_0"} 12.5`. +pub(super) fn answers_of(batches: &[Batch]) -> Result, String> { + let mut answers = BTreeMap::new(); + for batch in batches { + let schema = batch.schema(); + for row in batch.to_rows() { + let mut labels = SeriesLabels::new(); + let mut value = None; + for (field, cell) in schema.fields.iter().zip(&row) { + match cell { + Value::Utf8(text) if field.name == PROMQL_SERIES_IDENTITY => { + let identity: SeriesLabels = serde_json::from_str(text) + .map_err(|error| format!("series identity {text}: {error}"))?; + labels.extend(identity); + } + Value::Utf8(text) if !text.is_empty() => { + labels.insert(field.name.clone(), text.to_string()); + } + Value::Float64(number) if value.is_none() => value = Some(*number), + _ => {} + } + } + let value = value.ok_or_else(|| format!("an answer row without a value: {row:?}"))?; + answers.insert(labels, value); + } + } + Ok(answers) +} diff --git a/crates/devtools/src/bin/asap_run.rs b/crates/devtools/src/bin/asap_run.rs new file mode 100644 index 000000000..c6d8048da --- /dev/null +++ b/crates/devtools/src/bin/asap_run.rs @@ -0,0 +1,128 @@ +// asap_run: runs the Planner's executor as one lane of the engine comparison +// of sketch-bench's harness (`aqpbm-harness`). +// +// asap_run bench-external --lane asap --run-dir [--variant ]... +// [--core ] [--warmup ] [--runs ] [--max-core-busy ] +// +// Runs every (workload, variant) cell of the run directory another step +// prepared (`workloads/.json`, `plans//.plan.json` written by +// `asap_planner --plan-file`) on `asap_lane::AsapEngine`, which installs each +// plan file as given, and writes `results/asap/__.cell.json` and the +// arm results beside them, as `approxbench bench-external run` does for +// its lanes; `approxbench bench-external report` then joins every lane. +// - `--variant`: only these variants (repeatable); every one by default. +// - `--core`: the core each step is pinned to (default 0). +// - `--warmup`, `--runs`: runs discarded, and measured (odd), per step +// (the harness's defaults). +// - `--max-core-busy`: a core busier than this fraction before a step +// refuses it (the harness's default). +// Prints one line per cell. Exits 2 on a usage error, 1 when the lane +// cannot run (a cell that fails is written as not run, and the next runs). + +use std::path::PathBuf; +use std::process::ExitCode; + +use aqpbm_harness::bench_external::{run_lane, LaneSetup}; +use aqpbm_harness::engine::EngineUnderTest; +use aqpbm_harness::protocol::{ + load_probe_for, MeasurementClocks, OsCpuClock, ProcessCpuFiles, Protocol, +}; +use aqpbm_harness::rundir::RunDir; +use asap_devtools::asap_lane::AsapEngine; + +const USAGE: &str = + "usage: asap_run bench-external --lane asap --run-dir [--variant ]... \ +[--core ] [--warmup ] [--runs ] [--max-core-busy ]"; + +/// The lanes this binary runs. +const LANES: [&str; 1] = ["asap"]; + +enum Failure { + Usage(String), + Run(String), +} + +fn main() -> ExitCode { + match run(std::env::args().skip(1).collect()) { + Ok(()) => ExitCode::SUCCESS, + Err(Failure::Usage(error)) => { + eprintln!("asap_run: {error}\n{USAGE}"); + ExitCode::from(2) + } + Err(Failure::Run(error)) => { + eprintln!("asap_run: {error}"); + ExitCode::FAILURE + } + } +} + +fn run(args: Vec) -> Result<(), Failure> { + let mut args = args.into_iter(); + match args.next().as_deref() { + Some("bench-external") => {} + Some(other) => return Err(Failure::Usage(format!("unknown command {other}"))), + None => return Err(Failure::Usage("a command is required".into())), + } + let mut lane = None; + let mut run_dir = None; + let mut variants = Vec::new(); + let mut core = 0; + let mut protocol = Protocol::default(); + while let Some(flag) = args.next() { + let value = args + .next() + .ok_or_else(|| Failure::Usage(format!("{flag} needs a value")))?; + let number = |value: &str| -> Result { + value + .parse() + .map_err(|e| Failure::Usage(format!("{flag} {value}: {e}"))) + }; + match flag.as_str() { + "--lane" => lane = Some(value), + "--run-dir" => run_dir = Some(PathBuf::from(value)), + "--variant" => variants.push(value), + "--core" => core = number(&value)? as usize, + "--warmup" => protocol.warmup_runs = number(&value)?, + "--runs" => protocol.measured_runs = number(&value)?, + "--max-core-busy" => { + protocol.max_core_busy_fraction = value + .parse() + .map_err(|e| Failure::Usage(format!("{flag} {value}: {e}")))? + } + other => return Err(Failure::Usage(format!("unknown argument {other}"))), + } + } + let engine = match lane.as_deref() { + Some("asap") => AsapEngine, + Some(other) => { + return Err(Failure::Usage(format!( + "unknown lane {other}; known lanes: {}", + LANES.join(", ") + ))) + } + None => return Err(Failure::Usage("--lane is required".into())), + }; + let run_dir = run_dir.ok_or_else(|| Failure::Usage("--run-dir is required".into()))?; + let run_dir = RunDir::open(&run_dir).map_err(|e| Failure::Run(e.to_string()))?; + let clocks = MeasurementClocks::with_calling_thread(&OsCpuClock); + let mut probe = load_probe_for(&engine.cpu_accounting(), &ProcessCpuFiles::os()) + .map_err(|e| Failure::Run(e.to_string()))?; + let only = (!variants.is_empty()).then_some(variants.as_slice()); + let cells = run_lane( + &run_dir, + &engine, + only, + LaneSetup { + protocol: &protocol, + core, + probe: probe.as_mut(), + clocks: &clocks, + args: std::env::args().collect(), + }, + ) + .map_err(|e| Failure::Run(e.to_string()))?; + for cell in cells { + println!("{} {} {:?}", cell.workload, cell.variant, cell.outcome); + } + Ok(()) +} diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index de505a8cf..d2e1ae4c5 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -7,8 +7,11 @@ //! query corpora and holds their SQL catalogs. [`rqe_atomic_costs`] converts //! sketch-bench's measured per-operation costs into plan selection's //! measured table. [`planner_fuzz`] is the metamorphic fuzz tester for -//! missed reuse and sharing behind the `planner_fuzz` bin. +//! missed reuse and sharing behind the `planner_fuzz` bin. [`asap_lane`] is +//! the in-process ASAP lane of the engine comparison behind the `asap_run` +//! bin. +pub mod asap_lane; pub mod candidate_labels; pub mod corpora; pub mod planner_fuzz; diff --git a/crates/devtools/tests/asap_lane.rs b/crates/devtools/tests/asap_lane.rs index fe9d189ea..b0d1c14eb 100644 --- a/crates/devtools/tests/asap_lane.rs +++ b/crates/devtools/tests/asap_lane.rs @@ -151,8 +151,7 @@ fn run_dir(temp: &TempDir, workloads: &[&WorkloadSpec], plans: &[(&str, &str, Pa let run = RunDir::create_at(&temp.0.join("run")).unwrap(); for workload in workloads { write_json( - &run - .path() + &run.path() .join("workloads") .join(format!("{}.json", workload.name)), workload, @@ -171,7 +170,11 @@ fn run_dir(temp: &TempDir, workloads: &[&WorkloadSpec], plans: &[(&str, &str, Pa /// Runs the ASAP lane over `run` with warm-up `warmup` and `runs` measured /// runs. -fn run_asap_lane(run: &RunDir, warmup: u32, runs: u32) -> Vec { +fn run_asap_lane( + run: &RunDir, + warmup: u32, + runs: u32, +) -> Vec { let protocol = Protocol { warmup_runs: warmup, measured_runs: runs, @@ -313,7 +316,7 @@ fn conformance_kit_passes_on_the_asap_engine() { /// `sum_over_time(m[5m])` every 10 s over 2 series, planned twice: with /// samples arriving (a chain of 10 s tumbling windows built at ingestion /// time) and at rest (raw samples scanned at query time). The ASAP lane -/// runs both plans (one warm-up and two measured runs, so three installs +/// runs both plans (one warm-up and three measured runs, so four installs /// interleaved) and answers every scheduled evaluation exactly: per series, /// the sum of its samples in `(T - 5 min, T]`. #[test] @@ -334,7 +337,7 @@ fn the_asap_lane_runs_a_planned_sum_over_time_and_answers_exactly() { &[&workload], &[("ingesting", "w", ingesting), ("at-rest", "w", at_rest)], ); - let cells = run_asap_lane(&run, 1, 2); + let cells = run_asap_lane(&run, 1, 3); assert_eq!(cells.len(), 2, "{cells:?}"); assert!( cells.iter().all(|cell| cell.outcome == CellOutcome::Ran), @@ -429,10 +432,11 @@ fn a_plan_for_other_queries_is_refused_naming_the_query() { engine_options: options, }; let with_plan = arm(serde_json::json!({ "plan_file": plan_file })); - let install_error = |workload: &WorkloadSpec, arm: &ArmSpec| match AsapEngine.install(workload, arm) { - Ok(_) => panic!("installed"), - Err(EngineError(message)) => message, - }; + let install_error = + |workload: &WorkloadSpec, arm: &ArmSpec| match AsapEngine.install(workload, arm) { + Ok(_) => panic!("installed"), + Err(EngineError(message)) => message, + }; let workload = harness_workload("w", SUM_QUERY, 300_000); let error = install_error(&workload, &arm(serde_json::Value::Null)); @@ -442,7 +446,10 @@ fn a_plan_for_other_queries_is_refused_naming_the_query() { assert!(error.contains("missing.plan.json"), "{error}"); let not_a_plan = temp.0.join("not-a-plan.json"); std::fs::write(¬_a_plan, "{}").unwrap(); - let error = install_error(&workload, &arm(serde_json::json!({ "plan_file": not_a_plan }))); + let error = install_error( + &workload, + &arm(serde_json::json!({ "plan_file": not_a_plan })), + ); assert!(error.contains("not-a-plan.json"), "{error}"); // Another query text: the error names the query and both texts. @@ -458,7 +465,10 @@ fn a_plan_for_other_queries_is_refused_naming_the_query() { ..two.queries[0].clone() }); let error = install_error(&two, &with_plan); - assert!(error.contains("1 query text") && error.contains("2 queries"), "{error}"); + assert!( + error.contains("1 query text") && error.contains("2 queries"), + "{error}" + ); // The plan's text as the query's second text (after a SQL text) matches. let mut sql_first = workload.clone(); diff --git a/crates/executor/src/sources/memory.rs b/crates/executor/src/sources/memory.rs index ae4196444..687d1d8ec 100644 --- a/crates/executor/src/sources/memory.rs +++ b/crates/executor/src/sources/memory.rs @@ -2,28 +2,55 @@ use super::*; use planner_types::plan::physical_asap_dag::PhysicalASAPNodeId; use std::collections::BTreeMap; -/// An in-memory [`DataStore`], for tests and examples: raw tables by -/// `Source`, and each materialized view's rows by its node id, time-indexed -/// and append-only. A run's output for a view is appended with the time -/// range its rows cover: a chain's window `(window_end - w, window_end]`, -/// or the range of data another view's state summarizes. Appending drops the -/// rows older than the view's `retention_ms` before the newest appended -/// end. A read returns the rows whose time ranges tile the requested range -/// exactly, and fails when one is missing. Raw tables return every row (the -/// executor re-applies the raw data request). +/// An in-memory [`DataStore`]: raw tables by `Source`, and each +/// materialized view's rows by its node id, time-indexed and append-only. +/// A run's output for a view is appended with the time range its rows +/// cover: a chain's window `(window_end - w, window_end]`, or the range of +/// data another view's state summarizes. Appending drops the rows older +/// than the view's `retention_ms` before the newest appended end. A read +/// returns the rows whose time ranges tile the requested range exactly, and +/// fails when one is missing. +/// +/// A raw table is bound whole ([`Self::with_raw_data`]): every read returns +/// every row; or bound for appending ([`Self::with_appended_raw_data`]): a +/// runtime appends each batch of samples with the `(start, end]` it covers +/// as it arrives, a read returns the batches overlapping the requested +/// time range, and appending drops the batches older than the table's +/// lookback. Either way the executor re-applies the raw data request. /// /// The store owns the resident batches (a run is not charged for their /// states again); each cursor clones only the next batch, not the entire /// data set. #[derive(Default)] pub struct InMemoryDataStore { - raw_data: Vec<(Source, SchemaRef, Vec)>, + raw_data: Vec, /// Each view's appended rows, oldest first. materialized_views: BTreeMap>, } -/// Rows appended for a view: the `(start, end]` they cover and their batches. +/// Rows appended for a view or a raw table bound for appending: the +/// `(start, end]` they cover and their batches. type AppendedRows = ((i64, i64), Vec); + +/// The rows of one raw table. +struct RawTable { + source: Source, + schema: SchemaRef, + rows: RawRows, +} + +enum RawRows { + /// Bound whole: every read returns every row. + Whole(Vec), + /// Appended as samples arrive, oldest first; appending drops the rows + /// ending `lookback_ms` or more before the newest end (`None`: kept + /// forever). + Appended { + lookback_ms: Option, + rows: Vec, + }, +} + impl InMemoryDataStore { /// Bind `batches`, all in `schema`, as the rows of `source`. Bad input /// when `source` is already bound or a batch is in another schema. @@ -33,8 +60,107 @@ impl InMemoryDataStore { schema: SchemaRef, batches: Vec, ) -> Result { + self.bind_raw_data(source, schema, &batches)?; + held_by_store(&batches); + self.raw_data.last_mut().expect("just bound").rows = RawRows::Whole(batches); + Ok(self) + } + + /// Bind `source`, in `schema`, for appending its rows as they arrive + /// ([`Self::append_raw_data`]), keeping each batch until the newest + /// appended end is `lookback_ms` past its end (`None`: forever). Bad + /// input when `source` is already bound. Example: a runtime keeping + /// the last 5 minutes of a time series every query-time `Scan` of it + /// reads binds it with `Some(300_000)`. + pub fn with_appended_raw_data( + mut self, + source: Source, + schema: SchemaRef, + lookback_ms: Option, + ) -> Result { + self.bind_raw_data(source, schema, &[])?; + self.raw_data.last_mut().expect("just bound").rows = RawRows::Appended { + lookback_ms, + rows: Vec::new(), + }; + Ok(self) + } + + /// Append `batches`, all in the table's schema, as the rows of `source` + /// covering `time_range_ms` (`(start, end]`), then drop its batches + /// ending `lookback_ms` or more before `end` (the newest are always + /// kept). Bad input when `source` is not bound for appending, a batch + /// is in another schema, or the range is empty or starts before the end + /// of the rows appended last (the store is append-only). + pub fn append_raw_data( + &mut self, + source: &Source, + time_range_ms: (i64, i64), + batches: Vec, + ) -> Result<(), Error> { + let table = self + .raw_data + .iter_mut() + .find(|table| &table.source == source) + .ok_or_else(|| Error::Invalid(format!("unbound raw data: {source:?}")))?; + let RawRows::Appended { lookback_ms, rows } = &mut table.rows else { + return Err(Error::Invalid(format!( + "raw data {source:?} is bound whole, not for appending" + ))); + }; + if batches.iter().any(|batch| batch.schema() != &table.schema) { + return Err(Error::Invalid("raw data batch schema mismatch".into())); + } + let (start, end) = time_range_ms; + if start >= end + || rows + .last() + .is_some_and(|((_, last_end), _)| start < *last_end) + { + return Err(Error::Invalid(format!( + "raw data {source:?}: appended rows ({start}, {end}] are empty or not after the stored ones" + ))); + } + held_by_store(&batches); + rows.push((time_range_ms, batches)); + if let Some(lookback_ms) = *lookback_ms { + drop_rows_ending_before(rows, end, lookback_ms); + } + Ok(()) + } + + /// Bytes of the raw rows held. + pub fn raw_data_bytes(&self) -> usize { + self.raw_data + .iter() + .map(|table| match &table.rows { + RawRows::Whole(batches) => batches_bytes(batches), + RawRows::Appended { rows, .. } => { + rows.iter().map(|(_, batches)| batches_bytes(batches)).sum() + } + }) + .sum() + } + + /// Bytes of the materialized views' stored rows. + pub fn materialized_view_bytes(&self) -> usize { + self.materialized_views + .values() + .flatten() + .map(|(_, batches)| batches_bytes(batches)) + .sum() + } + + /// Bind `source` with no rows yet; bad input when it is already bound or + /// a batch of `batches` is in another schema than `schema`. + fn bind_raw_data( + &mut self, + source: Source, + schema: SchemaRef, + batches: &[Batch], + ) -> Result<(), Error> { crate::values::validate_schema(&schema)?; - if self.raw_data.iter().any(|(bound, _, _)| bound == &source) { + if self.raw_data.iter().any(|table| table.source == source) { return Err(Error::Invalid(format!( "raw data {source:?} is bound twice" ))); @@ -42,9 +168,12 @@ impl InMemoryDataStore { if batches.iter().any(|batch| batch.schema() != &schema) { return Err(Error::Invalid("raw data batch schema mismatch".into())); } - held_by_store(&batches); - self.raw_data.push((source, schema, batches)); - Ok(self) + self.raw_data.push(RawTable { + source, + schema, + rows: RawRows::Whole(Vec::new()), + }); + Ok(()) } /// Append `batches` as the rows of `view` covering `time_range_ms` /// (`(start, end]`), then drop its rows ending `view.retention_ms()` or @@ -71,13 +200,23 @@ impl InMemoryDataStore { } held_by_store(&batches); rows.push((time_range_ms, batches)); - let oldest_kept_end = - end.saturating_sub(i64::try_from(view.retention_ms()).unwrap_or(i64::MAX)); - rows.retain(|((_, row_end), _)| *row_end > oldest_kept_end || *row_end == end); + drop_rows_ending_before(rows, end, view.retention_ms()); Ok(()) } } +/// Drops the rows of `rows` ending `retention_ms` or more before `end`, +/// keeping those ending at `end`: retention 15 at end 30 drops `(0, 10]` +/// and keeps `(10, 20]`. +fn drop_rows_ending_before(rows: &mut Vec, end: i64, retention_ms: u64) { + let oldest_kept_end = end.saturating_sub(i64::try_from(retention_ms).unwrap_or(i64::MAX)); + rows.retain(|((_, row_end), _)| *row_end > oldest_kept_end || *row_end == end); +} + +fn batches_bytes(batches: &[Batch]) -> usize { + batches.iter().map(Batch::memory_bytes).sum() +} + fn held_by_store(batches: &[Batch]) { for batch in batches { batch.mark_states_held_outside_run(); @@ -108,17 +247,28 @@ impl DataStore for InMemoryDataStore { request: &RawDataRequest, context: RunContext, ) -> Result, Error> { - let (_, schema, batches) = self + let table = self .raw_data .iter() - .find(|(bound, _, _)| bound == source) + .find(|table| &table.source == source) .ok_or_else(|| Error::Invalid(format!("unbound raw data: {source:?}")))?; - let projection = projection(schema, request.schema)?; + let projection = projection(&table.schema, request.schema)?; let output = request.schema.clone(); + // An appended table returns the batches overlapping the requested + // range; the executor re-applies it row by row. + let batches: Box> = match (&table.rows, request.time_range_ms) { + (RawRows::Whole(batches), _) => Box::new(batches.iter()), + (RawRows::Appended { rows, .. }, None) => { + Box::new(rows.iter().flat_map(|(_, batches)| batches)) + } + (RawRows::Appended { rows, .. }, Some((start, end))) => Box::new( + rows.iter() + .filter(move |((row_start, row_end), _)| *row_end > start && *row_start < end) + .flat_map(|(_, batches)| batches), + ), + }; Ok(stream_batches( - batches - .iter() - .map(move |batch| batch.project(output.clone(), &projection)), + batches.map(move |batch| batch.project(output.clone(), &projection)), context, )) } diff --git a/crates/planner/src/installable/mod.rs b/crates/planner/src/installable/mod.rs index 5290f93ad..a7f9a77bf 100644 --- a/crates/planner/src/installable/mod.rs +++ b/crates/planner/src/installable/mod.rs @@ -26,6 +26,7 @@ use std::collections::{HashMap, HashSet}; use asap_cost::{NodeCostSource, OperatorKind, PlanCost}; +use asap_executor::physical_planner::CompiledPhysicalDAG; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::plan::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, @@ -308,6 +309,40 @@ impl InstallablePlan { executor_compiles(&self.dag, &views).map_err(|error| error.to_string()) } + /// The executor's query run of this plan, as a runtime runs it at each + /// evaluation: the roots, in workload order, over the views they read + /// (each a materialized view slot of the runtime's store) and the + /// `Scan`s they read outside them (raw data slots). `Err` is + /// [`InstallablePlanError::QueryRunNotExecutable`] when the executor + /// cannot compile it, or why the views cannot be derived. + pub fn compile_query_run(&self) -> Result { + let views = self.materialized_views()?; + let views_by_node = views.iter().map(|view| (view.node_id, view)).collect(); + executor_runs_queries(&IndexedDag::new(&self.dag), &views_by_node) + } + + /// The executor's refresh of `group`, one of + /// [`Self::ingestion_refresh_groups`], as a runtime runs it in one + /// ingestion run per window: the group's views as roots, in group + /// order, over the nodes below them, each view of another group they + /// read a materialized view slot. `Err` is + /// [`InstallablePlanError::MaterializedViewNotExecutable`] when the + /// executor cannot compile it, or why the views cannot be derived. + pub fn compile_ingestion_refresh( + &self, + group: &IngestionRefreshGroup, + ) -> Result { + let views = self.materialized_views()?; + let views_by_node = views.iter().map(|view| (view.node_id, view)).collect(); + let refreshed: Vec<&MaterializedView> = group.views.iter().collect(); + executor_refreshes( + &refreshed, + &views_by_node, + &IndexedDag::new(&self.dag), + false, + ) + } + /// The plan of `dag` answering `query_texts` (one per root, in workload /// order), checked as [`Self::validate`] checks a plan file: the DAG's /// validity, one text per root, no standalone scalar query, and each @@ -474,7 +509,7 @@ fn executor_compiles( for (position, refreshed) in refreshes.enumerate() { executor_refreshes(&refreshed, &views_by_node, &indexed, position == 0)?; } - executor_runs_queries(&indexed, &views_by_node) + executor_runs_queries(&indexed, &views_by_node).map(|_| ()) } /// Whether the executor compiles the query run: the roots over the views @@ -484,7 +519,7 @@ fn executor_compiles( fn executor_runs_queries( indexed: &IndexedDag<'_>, views_by_node: &HashMap, -) -> Result<(), InstallablePlanError> { +) -> Result { use asap_executor::physical_planner::compile; let dag = indexed.dag; let mut read = Vec::new(); @@ -500,10 +535,8 @@ fn executor_runs_queries( } } let roots: Vec = dag.roots.iter().map(|&id| id as u64).collect(); - compile(dag, &read, &roots).map(|_| ()).map_err(|error| { - InstallablePlanError::QueryRunNotExecutable { - reason: error.to_string(), - } + compile(dag, &read, &roots).map_err(|error| InstallablePlanError::QueryRunNotExecutable { + reason: error.to_string(), }) } @@ -516,7 +549,7 @@ fn executor_refreshes( views_by_node: &HashMap, indexed: &IndexedDag<'_>, in_whole_plan: bool, -) -> Result<(), InstallablePlanError> { +) -> Result { use asap_executor::physical_planner::compile; let roots: Vec = refreshed.iter().map(|view| view.node_id).collect(); // The refresh reads only the nodes below it: compile them, or the whole @@ -541,17 +574,15 @@ fn executor_refreshes( .filter_map(|id| views_by_node.get(id).map(|read| (*read).clone())) .collect(); let root_ids: Vec = roots.iter().map(|&id| id as u64).collect(); - compile(below, &read, &root_ids) - .map(|_| ()) - .map_err( - |error| InstallablePlanError::MaterializedViewNotExecutable { - node_id: roots[0], - reason: match roots.as_slice() { - [_] => error.to_string(), - _ => format!("refreshed with views {roots:?}: {error}"), - }, + compile(below, &read, &root_ids).map_err(|error| { + InstallablePlanError::MaterializedViewNotExecutable { + node_id: roots[0], + reason: match roots.as_slice() { + [_] => error.to_string(), + _ => format!("refreshed with views {roots:?}: {error}"), }, - ) + } + }) } /// The data facts the selected `plan` (costed as `cost`) assumed because diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index e3a8182ae..50bfb5f51 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -242,7 +242,11 @@ impl RawDataRetention { lookback_ms, } } +} +/// What a runtime keeps of each source: e.g. `m` with lookback 5 minutes, +/// the raw rows its query-time `Scan`s read back. +impl RawDataRetention { /// The source. pub fn source(&self) -> &Source { &self.source From 4ed579d1dcd3944d41310c5fddde69856db1d099 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:49:56 +0000 Subject: [PATCH 3/7] refactor: apply the independent review of the ASAP lane One raw-table schema rule (PhysicalASAPDAG::raw_data_schemas) for the lane and the planner tests' stand-in runtime; samples routed once per series, with dictionary texts cached per table; views' windows before the data start refreshed empty at install; two answer rows of one series are an error; no empty raw appends; simpler raw binding in the in-memory store. Pins sketch-bench at the rebased head (harness unchanged). --- Cargo.lock | 6 +- crates/devtools/Cargo.toml | 4 +- .../devtools/src/asap_lane/installed_plan.rs | 134 +++++----- .../src/asap_lane/time_series_batches.rs | 240 ++++++++++-------- crates/devtools/src/bin/asap_run.rs | 6 +- crates/executor/src/sources/memory.rs | 21 +- crates/executor/tests/raw_scan.rs | 23 +- crates/planner/tests/time_series/mod.rs | 37 +-- .../types/src/plan/materialized_view/mod.rs | 2 - .../src/plan/materialized_view/raw_data.rs | 32 ++- 10 files changed, 277 insertions(+), 228 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6ee11bb2..9453e38f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "aqpbm-core" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/sketch-bench?rev=af7fa57c917696815937f7cdd84f48c3391e1c89#af7fa57c917696815937f7cdd84f48c3391e1c89" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=671d3997b25cd03f79a927506a492485803d728b#671d3997b25cd03f79a927506a492485803d728b" dependencies = [ "anyhow", "aqpbm-datagen", @@ -66,7 +66,7 @@ dependencies = [ [[package]] name = "aqpbm-datagen" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/sketch-bench?rev=af7fa57c917696815937f7cdd84f48c3391e1c89#af7fa57c917696815937f7cdd84f48c3391e1c89" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=671d3997b25cd03f79a927506a492485803d728b#671d3997b25cd03f79a927506a492485803d728b" dependencies = [ "rand 0.9.5", "rand_distr", @@ -78,7 +78,7 @@ dependencies = [ [[package]] name = "aqpbm-harness" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/sketch-bench?rev=af7fa57c917696815937f7cdd84f48c3391e1c89#af7fa57c917696815937f7cdd84f48c3391e1c89" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=671d3997b25cd03f79a927506a492485803d728b#671d3997b25cd03f79a927506a492485803d728b" dependencies = [ "aqpbm-core", "aqpbm-datagen", diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index a000cb2d3..7c868be4e 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -24,7 +24,7 @@ asap-executor = { path = "../executor" } # --lane asap` runs the Planner's executor as an engine under test # (`asap_lane::AsapEngine`). Pinned to a sketch-bench commit; sketch-bench # never depends on the planner. -aqpbm-harness = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "af7fa57c917696815937f7cdd84f48c3391e1c89" } +aqpbm-harness = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "671d3997b25cd03f79a927506a492485803d728b" } futures = "0.3" # Used by the asap_planner / variant_coverage bins (catalog schemas, @@ -38,7 +38,7 @@ tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } [dev-dependencies] # The data generator of the harness's workloads, at the harness's pin. -aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "af7fa57c917696815937f7cdd84f48c3391e1c89" } +aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "671d3997b25cd03f79a927506a492485803d728b" } asap-types = { path = "../types", features = ["test-support"] } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } diff --git a/crates/devtools/src/asap_lane/installed_plan.rs b/crates/devtools/src/asap_lane/installed_plan.rs index 60ccb259c..ba91ab362 100644 --- a/crates/devtools/src/asap_lane/installed_plan.rs +++ b/crates/devtools/src/asap_lane/installed_plan.rs @@ -2,7 +2,6 @@ use std::sync::Arc; -use aqpbm_harness::data::Sample; use aqpbm_harness::engine::{ Answers, EngineError, Footprint, InstalledArm, PlanSummary, SampleChunk, StepReport, WorkloadSpec, @@ -15,9 +14,7 @@ use asap_planner::{InstallablePlan, MaterializedView, MaterializedViewRefresh, R use futures::executor::block_on; use futures::StreamExt; -use super::time_series_batches::{ - answers_of, time_series_tables, SeriesIdentities, TimeSeriesTable, -}; +use super::time_series_batches::{answers_of, time_series_tables, SampleRouter, TimeSeriesTable}; /// The memory one run may reserve: no limit in practice, so the lane /// measures the plan, not a limit of this runtime. @@ -31,19 +28,21 @@ pub(super) struct InstalledPlan { /// The workload query each root answers, in root order. query_ids: Vec, tables: Vec, - identities: SeriesIdentities, + router: SampleRouter, refreshes: Vec, query_run: CompiledPhysicalDAG, /// Shared with each run while it runs, held alone between steps. store: Arc, /// The end of the samples ingested so far: the data start before the - /// first chunk. + /// first chunk (the harness's samples lie in `(start, start + duration]`). ingested_until_ms: i64, } /// One ingestion refresh group: refreshed by one ingestion run per window /// `(e - width, e]` with `e + phase ≡ 0 (mod width)`, whose roots are its -/// views. +/// views. The phase is measured back from evaluation times on the width's +/// grid: the harness evaluates at multiples of the evaluation interval, +/// which plans made for evaluation phase 0 assume. struct GroupRefresh { window: RefreshWindow, views: Vec, @@ -104,7 +103,9 @@ impl InstalledPlan { let tables = time_series_tables(&plan.dag).map_err(unsupported)?; let retention = plan.raw_data_retention().map_err(not_executable)?; // A refresh reads the window ending at a chunk's end or before it, - // so the raw samples it reads reach back one window and one chunk. + // so the raw samples it reads reach back one window and one chunk; + // the harness's chunks are one evaluation interval each + // (`ArmSchedule::for_workload`). let chunk_ms = workload .queries .iter() @@ -134,47 +135,62 @@ impl InstalledPlan { .map(|record| record.description.start_ms) .min() .unwrap_or(0); + let mut store = Arc::new(store); + // The windows before the data that a view keeps, refreshed empty, + // so a query shifted back past the data start reads windows with + // no samples rather than missing ones. Example: a 10 s chain kept + // 5 min, with data from 0, has empty windows ending at -290 s ... + // 0 s before the first chunk. + for refresh in &refreshes { + let kept_ms = refresh + .views + .iter() + .map(MaterializedView::retention_ms) + .max(); + let from_ms = ingested_until_ms - kept_ms.unwrap_or(0) as i64; + for end_ms in window_ends(refresh.window, from_ms, ingested_until_ms) { + refresh_window(refresh, &mut store, end_ms)?; + } + } Ok(InstalledPlan { plan, query_ids, tables, - identities: SeriesIdentities::default(), + router: SampleRouter::default(), refreshes, query_run, - store: Arc::new(store), + store, ingested_until_ms, }) } +} - /// The store, between runs: no run holds it then. - fn store_mut(&mut self) -> &mut InMemoryDataStore { - Arc::get_mut(&mut self.store).expect("no run holds the store between steps") - } +/// The store, between runs: no run holds it then. +fn store_between_runs(store: &mut Arc) -> &mut InMemoryDataStore { + Arc::get_mut(store).expect("no run holds the store between steps") +} - /// Refreshes `refresh` for the window ending at `end_ms`: one ingestion - /// run whose roots are its views, each view's output appended. - fn refresh_window(&mut self, refresh: usize, end_ms: i64) -> Result<(), EngineError> { - let GroupRefresh { - window, - views, - compiled, - } = &self.refreshes[refresh]; - let start_ms = end_ms - window.width_ms as i64; - let scope = Scope::Ingestion { - window_start_ms: start_ms, - window_end_ms: end_ms, - }; - let outputs = run_ingestion(compiled, &self.store, scope) - .map_err(|error| run_failed(&format!("refresh of ({start_ms}, {end_ms}]"), error))?; - let views = views.clone(); - let store = self.store_mut(); - for (view, batches) in views.iter().zip(outputs) { - store - .append_materialized_view(view, (start_ms, end_ms), batches) - .map_err(|error| run_failed("storing a view", error))?; - } - Ok(()) +/// Refreshes `refresh` for the window ending at `end_ms`: one ingestion run +/// whose roots are its views, each view's output appended to `store`. +fn refresh_window( + refresh: &GroupRefresh, + store: &mut Arc, + end_ms: i64, +) -> Result<(), EngineError> { + let start_ms = end_ms - refresh.window.width_ms as i64; + let scope = Scope::Ingestion { + window_start_ms: start_ms, + window_end_ms: end_ms, + }; + let outputs = run_ingestion(&refresh.compiled, store, scope) + .map_err(|error| run_failed(&format!("refresh of ({start_ms}, {end_ms}]"), error))?; + let store = store_between_runs(store); + for (view, batches) in refresh.views.iter().zip(outputs) { + store + .append_materialized_view(view, (start_ms, end_ms), batches) + .map_err(|error| run_failed("storing a view", error))?; } + Ok(()) } /// `compiled` run in ingestion `scope` over `store`: each root's batches. @@ -288,38 +304,26 @@ impl InstalledArm for InstalledPlan { } /// Appends the chunk's samples of each time series the plan reads as - /// its rows for `(previous end, chunk end]`, then refreshes each group - /// for every window ending in that range. + /// its rows for `(previous end, chunk end]` (nothing for a series with + /// no sample in it), then refreshes each group for every window ending + /// in that range. fn ingest(&mut self, chunk: &SampleChunk) -> Result { let (after_ms, until_ms) = (self.ingested_until_ms, chunk.end_ms); - let mut batches = Vec::with_capacity(self.tables.len()); - for table in &self.tables { - let samples: Vec<&Sample> = chunk - .samples - .iter() - .filter(|sample| sample.labels.get("__name__") == Some(&table.metric)) - .collect(); + let routed = self.router.route(chunk, &mut self.tables); + for (table, samples) in self.tables.iter_mut().zip(routed) { + if samples.is_empty() { + continue; + } let batch = table - .batch(&samples, &mut self.identities) + .batch(&samples) .map_err(|error| run_failed("raw samples", error))?; - batches.push(batch); - } - let sources: Vec<_> = self - .tables - .iter() - .map(|table| table.source.clone()) - .collect(); - let store = self.store_mut(); - for (source, batch) in sources.iter().zip(batches) { - store - .append_raw_data(source, (after_ms, until_ms), vec![batch]) + store_between_runs(&mut self.store) + .append_raw_data(&table.source, (after_ms, until_ms), vec![batch]) .map_err(|error| run_failed("raw samples", error))?; } - for refresh in 0..self.refreshes.len() { - let ends: Vec = - window_ends(self.refreshes[refresh].window, after_ms, until_ms).collect(); - for end_ms in ends { - self.refresh_window(refresh, end_ms)?; + for refresh in &self.refreshes { + for end_ms in window_ends(refresh.window, after_ms, until_ms) { + refresh_window(refresh, &mut self.store, end_ms)?; } } self.ingested_until_ms = until_ms; @@ -338,7 +342,9 @@ impl InstalledArm for InstalledPlan { Ok((answers, StepReport::default())) } - /// The bytes of the stored views and of the raw samples held. + /// The bytes of the stored views and of the raw samples held, as the + /// executor counts each batch: a dictionary of series texts shared by + /// several batches is counted in each. fn footprint(&self) -> Result { Ok(Footprint { view_bytes: self.store.materialized_view_bytes() as u64, diff --git a/crates/devtools/src/asap_lane/time_series_batches.rs b/crates/devtools/src/asap_lane/time_series_batches.rs index ccf10e8f3..0f8a08881 100644 --- a/crates/devtools/src/asap_lane/time_series_batches.rs +++ b/crates/devtools/src/asap_lane/time_series_batches.rs @@ -10,7 +10,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use aqpbm_harness::data::{Sample, SeriesLabels}; +use aqpbm_harness::data::{Sample, SampleChunk, SeriesLabels}; use asap_executor::arrow::array::{ ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array, }; @@ -18,18 +18,22 @@ use asap_executor::arrow::datatypes::UInt32Type; use asap_executor::values::{Batch, BatchColumn, SchemaRef, Value}; use asap_planner::PhysicalASAPDAG; use asap_types::ir::operator::Source; -use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; -use asap_types::ir::NonASAPOp; -use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; +use asap_types::ir::schema::{DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; -/// One time series' raw table: what each of its columns holds. +/// One time series' raw table: what each of its columns holds, and the +/// series seen so far, each with an ordinal that is its dictionary key in +/// every text column. pub(super) struct TimeSeriesTable { - /// `Source::TimeSeries` of `metric`. + /// `Source::TimeSeries` of its metric. pub(super) source: Source, - pub(super) metric: String, - /// The union of its `Scan`s' columns, by name. + /// The union of its `Scan`s' columns. pub(super) schema: SchemaRef, columns: Vec, + /// The series seen so far, by ordinal. + series: Vec>, + /// Each column's dictionary texts, one per series of `series` (`None` + /// for a column that is not text); rebuilt when a series is added. + texts: Option>>, } /// What one column of a time series' raw table holds. @@ -44,33 +48,17 @@ enum TimeSeriesColumn { Label(String), } -/// The raw table of each time series `dag`'s `Scan`s read, in metric order. -/// `Err` for a source the lane cannot feed: a SQL table, or a column that -/// is neither the time, the value, the series identity nor a text label. +/// The raw table of each time series `dag`'s `Scan`s read, in the order of +/// its first `Scan` ([`PhysicalASAPDAG::raw_data_schemas`]). `Err` for a +/// source the lane cannot feed: a SQL table, or a column that is neither +/// the time, the value, the series identity nor a text label. pub(super) fn time_series_tables(dag: &PhysicalASAPDAG) -> Result, String> { - let mut schemas: BTreeMap<&str, Schema> = BTreeMap::new(); - for node in &dag.nodes { - let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, schema, .. }) = - &node.payload - else { - continue; - }; - let Source::TimeSeries { metric } = source else { - return Err(format!( - "node {} scans {source:?}, not a time series", - node.id - )); - }; - let known = schemas.entry(metric).or_insert_with(|| schema.clone()); - for field in &schema.fields { - if !known.fields.iter().any(|known| known.name == field.name) { - known.fields.push(field.clone()); - } - } - } - schemas + dag.raw_data_schemas() .into_iter() - .map(|(metric, schema)| { + .map(|(source, schema)| { + if !matches!(source, Source::TimeSeries { .. }) { + return Err(format!("the plan scans {source:?}, not a time series")); + } let columns = schema .fields .iter() @@ -87,100 +75,133 @@ pub(super) fn time_series_tables(dag: &PhysicalASAPDAG) -> Result Err(format!( - "time series {metric}: column {name} of type {dtype:?} is not a label" + "{source:?}: column {name} of type {dtype:?} is not a label" )), }) .collect::>()?; Ok(TimeSeriesTable { - source: Source::TimeSeries { - metric: metric.to_string(), - }, - metric: metric.to_string(), + source, schema: Arc::new(schema), columns, + series: Vec::new(), + texts: None, }) }) .collect() } -/// Each series' identity text, computed once per series: the harness shares -/// one label map among a series' samples, so the map's address names the -/// series. The map is kept alive here, so its address is never reused by -/// another series. +/// Where each series' samples go: its table and its ordinal there, or +/// nowhere (a metric the plan does not read), looked up once per series. +/// The harness shares one label map among a series' samples, so the map's +/// address names the series; the map is kept alive here, so its address is +/// never reused by another series. #[derive(Default)] -pub(super) struct SeriesIdentities { - by_address: HashMap<*const SeriesLabels, (Arc, Arc)>, +pub(super) struct SampleRouter { + by_address: HashMap<*const SeriesLabels, Option<(usize, u32)>>, + /// Every series seen, keeping its address taken. + seen: Vec>, } -impl SeriesIdentities { - fn identity(&mut self, labels: &Arc) -> Arc { - let (_, identity) = self - .by_address - .entry(Arc::as_ptr(labels)) - .or_insert_with(|| { - let json = serde_json::to_string(labels.as_ref()).expect("labels are JSON"); - (Arc::clone(labels), json.into()) - }); - Arc::clone(identity) +impl SampleRouter { + /// `chunk`'s samples per table of `tables`, each with its series' + /// ordinal, in chunk order; a sample of a metric no table reads is + /// dropped. + pub(super) fn route<'c>( + &mut self, + chunk: &'c SampleChunk, + tables: &mut [TimeSeriesTable], + ) -> Vec> { + let mut routed = vec![Vec::new(); tables.len()]; + for sample in &chunk.samples { + let route = *self + .by_address + .entry(Arc::as_ptr(&sample.labels)) + .or_insert_with(|| { + self.seen.push(Arc::clone(&sample.labels)); + let metric = sample.labels.get("__name__")?; + let table = tables.iter().position(|table| { + matches!(&table.source, Source::TimeSeries { metric: read } if read == metric) + })?; + Some((table, tables[table].add_series(&sample.labels))) + }); + if let Some((table, ordinal)) = route { + routed[table].push((ordinal, sample)); + } + } + routed } } impl TimeSeriesTable { - /// The rows of `samples` (all of this table's metric) in the table's - /// schema. Every text column is dictionary-encoded over the chunk's - /// series: one key per row, one text per series. + /// Adds a series; its ordinal. + fn add_series(&mut self, labels: &Arc) -> u32 { + self.series.push(Arc::clone(labels)); + self.texts = None; + (self.series.len() - 1) as u32 + } + + /// Each text column's dictionary texts, one per series, built when a + /// series was added since the last call. + fn texts(&mut self) -> &[Option] { + let series = &self.series; + self.texts.get_or_insert_with(|| { + self.columns + .iter() + .map(|column| { + let texts: StringArray = match column { + TimeSeriesColumn::Time | TimeSeriesColumn::Value => return None, + TimeSeriesColumn::SeriesIdentity => series + .iter() + .map(|labels| { + Some( + serde_json::to_string(labels.as_ref()) + .expect("labels are JSON"), + ) + }) + .collect(), + TimeSeriesColumn::Label(label) => series + .iter() + .map(|labels| Some(labels.get(label).map_or("", String::as_str))) + .collect(), + }; + Some(Arc::new(texts) as ArrayRef) + }) + .collect() + }) + } + + /// The rows of `samples` (this table's, each with its series' ordinal) + /// in the table's schema. Every text column is dictionary-encoded: the + /// series' ordinal is the key, and the dictionary holds every series + /// seen so far, shared by every batch until a series is added. pub(super) fn batch( - &self, - samples: &[&Sample], - identities: &mut SeriesIdentities, + &mut self, + samples: &[(u32, &Sample)], ) -> Result { - // Each row's series as an index into this chunk's series. - let mut series: Vec<&Arc> = Vec::new(); - let mut index_of: HashMap<*const SeriesLabels, u32> = HashMap::new(); - let keys: UInt32Array = samples - .iter() - .map(|sample| { - *index_of - .entry(Arc::as_ptr(&sample.labels)) - .or_insert_with(|| { - series.push(&sample.labels); - (series.len() - 1) as u32 - }) - }) - .collect(); - let text_column = |texts: StringArray| -> Result { - let column = DictionaryArray::::try_new(keys.clone(), Arc::new(texts)) - .map_err(|error| asap_executor::Error::Invalid(error.to_string()))?; - Ok(BatchColumn::Plain(Arc::new(column) as ArrayRef)) - }; - let columns = self - .columns - .iter() - .map(|column| match column { - TimeSeriesColumn::Time => Ok(BatchColumn::Plain(Arc::new( - TimestampMillisecondArray::from_iter_values( - samples.iter().map(|sample| sample.ts_ms), - ), - ))), - TimeSeriesColumn::Value => Ok(BatchColumn::Plain(Arc::new( - Float64Array::from_iter_values(samples.iter().map(|sample| sample.value)), - ))), - TimeSeriesColumn::SeriesIdentity => { - let texts: Vec> = series - .iter() - .map(|labels| identities.identity(labels)) - .collect(); - text_column(texts.iter().map(|text| Some(text.as_ref())).collect()) + let keys = UInt32Array::from_iter_values(samples.iter().map(|(ordinal, _)| *ordinal)); + let schema = self.schema.clone(); + let mut columns = Vec::with_capacity(self.columns.len()); + // The texts are `Arc`s: cloning them shares the dictionary. + let texts = self.texts().to_vec(); + for (column, texts) in self.columns.iter().zip(texts) { + columns.push(match (column, texts) { + (TimeSeriesColumn::Time, _) => { + BatchColumn::Plain(Arc::new(TimestampMillisecondArray::from_iter_values( + samples.iter().map(|(_, sample)| sample.ts_ms), + ))) } - TimeSeriesColumn::Label(label) => text_column( - series - .iter() - .map(|labels| Some(labels.get(label).map_or("", String::as_str))) - .collect(), - ), - }) - .collect::, _>>()?; - Batch::try_from_columns(self.schema.clone(), columns) + (TimeSeriesColumn::Value, _) => BatchColumn::Plain(Arc::new( + Float64Array::from_iter_values(samples.iter().map(|(_, sample)| sample.value)), + )), + (_, texts) => { + let texts = texts.expect("a text column has its texts"); + let column = DictionaryArray::::try_new(keys.clone(), texts) + .map_err(|error| asap_executor::Error::Invalid(error.to_string()))?; + BatchColumn::Plain(Arc::new(column)) + } + }); + } + Batch::try_from_columns(schema, columns) } } @@ -189,6 +210,7 @@ impl TimeSeriesTable { /// one, and every other non-empty text column (a label the query grouped /// by); the value is the row's first Float64 column. Example: a row /// `(ts, 12.5, {"s":"s_0"})` of `sum_over_time(m[5m])` is `{s="s_0"} 12.5`. +/// `Err` for two rows of one series, which a query answer never has. pub(super) fn answers_of(batches: &[Batch]) -> Result, String> { let mut answers = BTreeMap::new(); for batch in batches { @@ -211,7 +233,11 @@ pub(super) fn answers_of(batches: &[Batch]) -> Result) -> Result<(), Failure> { Some("asap") => AsapEngine, Some(other) => { return Err(Failure::Usage(format!( - "unknown lane {other}; known lanes: {}", - LANES.join(", ") + "unknown lane {other}; known lanes: asap" ))) } None => return Err(Failure::Usage("--lane is required".into())), diff --git a/crates/executor/src/sources/memory.rs b/crates/executor/src/sources/memory.rs index 687d1d8ec..14fd571b3 100644 --- a/crates/executor/src/sources/memory.rs +++ b/crates/executor/src/sources/memory.rs @@ -60,9 +60,11 @@ impl InMemoryDataStore { schema: SchemaRef, batches: Vec, ) -> Result { - self.bind_raw_data(source, schema, &batches)?; + if batches.iter().any(|batch| batch.schema() != &schema) { + return Err(Error::Invalid("raw data batch schema mismatch".into())); + } held_by_store(&batches); - self.raw_data.last_mut().expect("just bound").rows = RawRows::Whole(batches); + self.bind_raw_data(source, schema, RawRows::Whole(batches))?; Ok(self) } @@ -78,11 +80,11 @@ impl InMemoryDataStore { schema: SchemaRef, lookback_ms: Option, ) -> Result { - self.bind_raw_data(source, schema, &[])?; - self.raw_data.last_mut().expect("just bound").rows = RawRows::Appended { + let rows = RawRows::Appended { lookback_ms, rows: Vec::new(), }; + self.bind_raw_data(source, schema, rows)?; Ok(self) } @@ -151,13 +153,13 @@ impl InMemoryDataStore { .sum() } - /// Bind `source` with no rows yet; bad input when it is already bound or - /// a batch of `batches` is in another schema than `schema`. + /// Bind `source` to `rows`, all in `schema`; bad input when it is + /// already bound. fn bind_raw_data( &mut self, source: Source, schema: SchemaRef, - batches: &[Batch], + rows: RawRows, ) -> Result<(), Error> { crate::values::validate_schema(&schema)?; if self.raw_data.iter().any(|table| table.source == source) { @@ -165,13 +167,10 @@ impl InMemoryDataStore { "raw data {source:?} is bound twice" ))); } - if batches.iter().any(|batch| batch.schema() != &schema) { - return Err(Error::Invalid("raw data batch schema mismatch".into())); - } self.raw_data.push(RawTable { source, schema, - rows: RawRows::Whole(Vec::new()), + rows, }); Ok(()) } diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 638e8cb7d..10561b58b 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -1063,14 +1063,23 @@ fn appended_raw_data_is_read_by_range_and_dropped_after_its_lookback() { .append_raw_data(&source, (start, end), vec![rows_at(&schema, &[end])]) .unwrap(); } - assert_eq!(times_read(&store, &source, &schema, Some((5, 20))), [10, 20]); + assert_eq!( + times_read(&store, &source, &schema, Some((5, 20))), + [10, 20] + ); assert_eq!(times_read(&store, &source, &schema, Some((10, 20))), [20]); store .append_raw_data(&source, (20, 30), vec![rows_at(&schema, &[25, 30])]) .unwrap(); assert_eq!(times_read(&store, &source, &schema, None), [20, 25, 30]); - assert_eq!(times_read(&store, &source, &schema, Some((15, 30))), [20, 25, 30]); - assert_eq!(times_read(&store, &source, &schema, Some((20, 30))), [25, 30]); + assert_eq!( + times_read(&store, &source, &schema, Some((15, 30))), + [20, 25, 30] + ); + assert_eq!( + times_read(&store, &source, &schema, Some((20, 30))), + [25, 30] + ); let held = rows_at(&schema, &[20]).memory_bytes() + rows_at(&schema, &[25, 30]).memory_bytes(); assert_eq!(store.raw_data_bytes(), held); assert_eq!(store.materialized_view_bytes(), 0); @@ -1079,11 +1088,15 @@ fn appended_raw_data_is_read_by_range_and_dropped_after_its_lookback() { assert!(store .append_raw_data(&source, (25, 40), vec![rows_at(&schema, &[40])]) .is_err()); - let other = Source::TimeSeries { metric: "other".into() }; + let other = Source::TimeSeries { + metric: "other".into(), + }; assert!(store .append_raw_data(&other, (30, 40), vec![rows_at(&schema, &[40])]) .is_err()); - let fixed = Source::TimeSeries { metric: "fixed".into() }; + let fixed = Source::TimeSeries { + metric: "fixed".into(), + }; let mut store = store .with_raw_data(fixed.clone(), schema.clone(), vec![rows_at(&schema, &[1])]) .unwrap(); diff --git a/crates/planner/tests/time_series/mod.rs b/crates/planner/tests/time_series/mod.rs index 0be819d9b..2f808444f 100644 --- a/crates/planner/tests/time_series/mod.rs +++ b/crates/planner/tests/time_series/mod.rs @@ -17,8 +17,7 @@ use asap_executor::Error; use asap_planner::{InstallablePlan, MaterializedView}; use asap_types::ir::operator::Source; use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; -use asap_types::ir::NonASAPOp; -use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use futures::{executor::block_on, StreamExt}; /// One raw data sample as it arrives, e.g. one Prometheus Remote Write sample. @@ -67,40 +66,24 @@ pub fn series_identity(labels: &BTreeMap) -> String { /// every column those `Scan`s read. pub fn time_series_tables(dag: &PhysicalASAPDAG, samples: &[RawDataSample]) -> InMemoryDataStore { // Each metric's columns: the union of its Scans' columns, by name. - let mut schemas: BTreeMap<&str, Schema> = BTreeMap::new(); - for node in &dag.nodes { - let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric }, - schema, - .. - }) = &node.payload - else { - continue; - }; - let known = schemas.entry(metric).or_insert_with(|| schema.clone()); - for field in &schema.fields { - if !known.fields.iter().any(|f| f.name == field.name) { - known.fields.push(field.clone()); - } - } - } + let schemas = dag + .raw_data_schemas() + .into_iter() + .filter_map(|(source, schema)| match source { + Source::TimeSeries { metric } => Some((metric, schema)), + Source::Table { .. } => None, + }); let mut tables = InMemoryDataStore::default(); for (metric, schema) in schemas { let schema = Arc::new(schema); let rows = samples .iter() - .filter(|s| s.labels.get("__name__").map(String::as_str) == Some(metric)) + .filter(|s| s.labels.get("__name__") == Some(&metric)) .map(|sample| row(&schema, sample)) .collect(); let batch = Batch::try_new(schema.clone(), rows).unwrap(); tables = tables - .with_raw_data( - Source::TimeSeries { - metric: metric.to_string(), - }, - schema, - vec![batch], - ) + .with_raw_data(Source::TimeSeries { metric }, schema, vec![batch]) .unwrap(); } tables diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index 50bfb5f51..53476da6d 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -244,8 +244,6 @@ impl RawDataRetention { } } -/// What a runtime keeps of each source: e.g. `m` with lookback 5 minutes, -/// the raw rows its query-time `Scan`s read back. impl RawDataRetention { /// The source. pub fn source(&self) -> &Source { diff --git a/crates/types/src/plan/materialized_view/raw_data.rs b/crates/types/src/plan/materialized_view/raw_data.rs index b17f8ac1e..30e2c8a27 100644 --- a/crates/types/src/plan/materialized_view/raw_data.rs +++ b/crates/types/src/plan/materialized_view/raw_data.rs @@ -1,13 +1,16 @@ //! How far back a runtime keeps each raw-data source for the DAG's raw-data //! scans ([`PhysicalASAPDAG::raw_data_retention`]), which nodes keep rows -//! ([`PhysicalASAPDAGNode::retains_rows`]), and the shape checks of every -//! raw-data window ([`PhysicalASAPDAG::check_raw_data_windows`]). +//! ([`PhysicalASAPDAGNode::retains_rows`]), the shape checks of every +//! raw-data window ([`PhysicalASAPDAG::check_raw_data_windows`]), and the +//! columns a runtime keeps of each source ([`PhysicalASAPDAG::raw_data_schemas`]). use super::views::ViewIndex; use super::{InstallableDagError, RawDataRetention}; +use crate::ir::operator::arguments::Source; use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; +use crate::ir::schema::Schema; use crate::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId}; use crate::stages::Lifetime; @@ -55,6 +58,31 @@ impl PhysicalASAPDAG { Ok(retention) } + /// Each raw-data source the DAG's `Scan`s read (at query or ingestion + /// time), in the order of its first `Scan`, with the union of those + /// `Scan`s' columns by name, in first-seen order: the raw table a + /// runtime binds for the source. Example: `sum by (job) (m)` and + /// `sum_over_time(m[5m])` together read `m` as `(ts, value, + /// $promql_series_identity, job)`. + pub fn raw_data_schemas(&self) -> Vec<(Source, Schema)> { + let mut schemas: Vec<(Source, Schema)> = Vec::new(); + for node in &self.nodes { + let Operator::NonASAP(NonASAPOp::Scan { source, schema, .. }) = &node.payload else { + continue; + }; + let Some((_, known)) = schemas.iter_mut().find(|(known, _)| known == source) else { + schemas.push((source.clone(), schema.clone())); + continue; + }; + for field in &schema.fields { + if !known.fields.iter().any(|known| known.name == field.name) { + known.fields.push(field.clone()); + } + } + } + schemas + } + /// Checks that every raw-data window the DAG reads (each `TimeRange`, /// and each `SummaryTumblingWindowChain`'s tumbling window, reading a /// `Scan` without passing another `TimeRange`) has a supported shape From bfa96a1154ba674bc5d0e3b17c242e2923ce5c47 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:52:34 +0000 Subject: [PATCH 4/7] test: absent_over_time as the ASAP lane's executor-only example last_over_time now compiles on the executor; absent_over_time is still planned but not compiled. --- crates/devtools/tests/asap_lane.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/devtools/tests/asap_lane.rs b/crates/devtools/tests/asap_lane.rs index b0d1c14eb..b05ef1199 100644 --- a/crates/devtools/tests/asap_lane.rs +++ b/crates/devtools/tests/asap_lane.rs @@ -388,18 +388,18 @@ fn the_asap_lane_runs_a_planned_sum_over_time_and_answers_exactly() { } } -/// `last_over_time(m[10s])`: Stages 1-3 plan it, but the Planner's executor -/// has no `last_over_time` kernel. The plan file written for a runtime other +/// `absent_over_time(m[10s])`: Stages 1-3 plan it, but the Planner's +/// executor cannot compile it. The plan file written for a runtime other /// than the executor (`--executor-check skip`) is a "not executable" cell /// on the ASAP lane, so the other lanes still run that plan. #[test] fn a_plan_the_executor_cannot_compile_is_a_not_executable_cell() { - const QUERY: &str = "last_over_time(m[10s])"; - let temp = TempDir::new("last"); + const QUERY: &str = "absent_over_time(m[10s])"; + let temp = TempDir::new("absent"); let planning = planning_workload(QUERY, "m", 2, 1_000, r#""at_rest""#); - let plan_file = temp.0.join("last.plan.json"); + let plan_file = temp.0.join("absent.plan.json"); // Precondition: an executor-only gap. Once the executor runs - // `last_over_time`, pick another query it plans but cannot compile. + // `absent_over_time`, pick another query it plans but cannot compile. assert!(!write_plan_file(&planning, &plan_file, "compile")); assert!(write_plan_file(&planning, &plan_file, "skip")); let workload = harness_workload("w", QUERY, 10_000); From 6ff6c25e07a6b02f0293cd5c4e6c5cf9ba487e70 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:18:36 +0000 Subject: [PATCH 5/7] build: the harness from aqpbm-core sketch-bench moved its harness into aqpbm-core (aqpbm_core::harness); pin aqpbm-core without default features (no sketch library, polars or CLI crate) at the bench-external head. --- Cargo.lock | 21 +++------------ crates/devtools/Cargo.toml | 7 ++--- .../devtools/src/asap_lane/installed_plan.rs | 2 +- crates/devtools/src/asap_lane/mod.rs | 4 +-- .../src/asap_lane/time_series_batches.rs | 2 +- crates/devtools/src/bin/asap_run.rs | 10 +++---- crates/devtools/tests/asap_lane.rs | 26 ++++++++++--------- 7 files changed, 31 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9453e38f6..ff4a7412a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "aqpbm-core" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/sketch-bench?rev=671d3997b25cd03f79a927506a492485803d728b#671d3997b25cd03f79a927506a492485803d728b" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae#7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae" dependencies = [ "anyhow", "aqpbm-datagen", @@ -61,12 +61,13 @@ dependencies = [ "rand_xoshiro", "serde", "serde_json", + "thiserror 2.0.18", ] [[package]] name = "aqpbm-datagen" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/sketch-bench?rev=671d3997b25cd03f79a927506a492485803d728b#671d3997b25cd03f79a927506a492485803d728b" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae#7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae" dependencies = [ "rand 0.9.5", "rand_distr", @@ -75,20 +76,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "aqpbm-harness" -version = "0.1.0" -source = "git+https://github.com/ProjectASAP/sketch-bench?rev=671d3997b25cd03f79a927506a492485803d728b#671d3997b25cd03f79a927506a492485803d728b" -dependencies = [ - "aqpbm-core", - "aqpbm-datagen", - "chrono", - "libc", - "serde", - "serde_json", - "thiserror 2.0.18", -] - [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -348,8 +335,8 @@ dependencies = [ name = "asap-devtools" version = "0.1.0" dependencies = [ + "aqpbm-core", "aqpbm-datagen", - "aqpbm-harness", "asap-cost", "asap-executor", "asap-frontend-metricsql", diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 7c868be4e..931ac0c74 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -20,11 +20,12 @@ asap-planner = { path = "../planner" } # The executor's supported summary families, which `asap_planner` plans # with when the deployment declares none, as `plan_installable` does. asap-executor = { path = "../executor" } -# The engine-comparison harness of sketch-bench: `asap_run bench-external +# The engine-comparison harness of sketch-bench (`aqpbm_core::harness`; +# no default features: no sketch library, polars or CLI crate): `asap_run bench-external # --lane asap` runs the Planner's executor as an engine under test # (`asap_lane::AsapEngine`). Pinned to a sketch-bench commit; sketch-bench # never depends on the planner. -aqpbm-harness = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "671d3997b25cd03f79a927506a492485803d728b" } +aqpbm-core = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae", default-features = false } futures = "0.3" # Used by the asap_planner / variant_coverage bins (catalog schemas, @@ -38,7 +39,7 @@ tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } [dev-dependencies] # The data generator of the harness's workloads, at the harness's pin. -aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "671d3997b25cd03f79a927506a492485803d728b" } +aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "7c0d121192beaf3b68b1b1ec36aab8aafc90d0ae" } asap-types = { path = "../types", features = ["test-support"] } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } diff --git a/crates/devtools/src/asap_lane/installed_plan.rs b/crates/devtools/src/asap_lane/installed_plan.rs index ba91ab362..75d8c8890 100644 --- a/crates/devtools/src/asap_lane/installed_plan.rs +++ b/crates/devtools/src/asap_lane/installed_plan.rs @@ -2,7 +2,7 @@ use std::sync::Arc; -use aqpbm_harness::engine::{ +use aqpbm_core::harness::engine::{ Answers, EngineError, Footprint, InstalledArm, PlanSummary, SampleChunk, StepReport, WorkloadSpec, }; diff --git a/crates/devtools/src/asap_lane/mod.rs b/crates/devtools/src/asap_lane/mod.rs index 31d6a9ba0..9c988c867 100644 --- a/crates/devtools/src/asap_lane/mod.rs +++ b/crates/devtools/src/asap_lane/mod.rs @@ -1,6 +1,6 @@ //! The in-process ASAP lane of the engine comparison: the Planner's //! executor as an engine under test of the sketch-bench harness -//! (`aqpbm-harness`). +//! (`aqpbm_core::harness`). //! //! [`AsapEngine`] installs a plan file `asap_planner --plan-file` wrote (an //! [`InstallablePlan`]) as given, so this lane and every other lane of a @@ -21,7 +21,7 @@ mod time_series_batches; use std::path::PathBuf; -use aqpbm_harness::engine::{ +use aqpbm_core::harness::engine::{ ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, WorkloadSpec, }; diff --git a/crates/devtools/src/asap_lane/time_series_batches.rs b/crates/devtools/src/asap_lane/time_series_batches.rs index 0f8a08881..3ca5e72e2 100644 --- a/crates/devtools/src/asap_lane/time_series_batches.rs +++ b/crates/devtools/src/asap_lane/time_series_batches.rs @@ -10,7 +10,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use aqpbm_harness::data::{Sample, SampleChunk, SeriesLabels}; +use aqpbm_core::harness::data::{Sample, SampleChunk, SeriesLabels}; use asap_executor::arrow::array::{ ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array, }; diff --git a/crates/devtools/src/bin/asap_run.rs b/crates/devtools/src/bin/asap_run.rs index 7b96524fe..40e633cb9 100644 --- a/crates/devtools/src/bin/asap_run.rs +++ b/crates/devtools/src/bin/asap_run.rs @@ -1,5 +1,5 @@ // asap_run: runs the Planner's executor as one lane of the engine comparison -// of sketch-bench's harness (`aqpbm-harness`). +// of sketch-bench's harness (`aqpbm_core::harness`). // // asap_run bench-external --lane asap --run-dir [--variant ]... // [--core ] [--warmup ] [--runs ] [--max-core-busy ] @@ -22,12 +22,12 @@ use std::path::PathBuf; use std::process::ExitCode; -use aqpbm_harness::bench_external::{run_lane, LaneSetup}; -use aqpbm_harness::engine::EngineUnderTest; -use aqpbm_harness::protocol::{ +use aqpbm_core::harness::bench_external::{run_lane, LaneSetup}; +use aqpbm_core::harness::engine::EngineUnderTest; +use aqpbm_core::harness::protocol::{ load_probe_for, MeasurementClocks, OsCpuClock, ProcessCpuFiles, Protocol, }; -use aqpbm_harness::rundir::RunDir; +use aqpbm_core::harness::rundir::RunDir; use asap_devtools::asap_lane::AsapEngine; const USAGE: &str = diff --git a/crates/devtools/tests/asap_lane.rs b/crates/devtools/tests/asap_lane.rs index b05ef1199..836ba302e 100644 --- a/crates/devtools/tests/asap_lane.rs +++ b/crates/devtools/tests/asap_lane.rs @@ -1,23 +1,25 @@ //! The in-process ASAP lane of the engine comparison: `AsapEngine` installs a //! plan file `asap_planner --plan-file` wrote and runs it with the Planner's -//! executor under the harness of sketch-bench (`aqpbm-harness`), as one +//! executor under the harness of sketch-bench (`aqpbm_core::harness`), as one //! lane of `asap_run bench-external`. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::process::Command; -use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, UniformParameter}; -use aqpbm_harness::arm_run::ArmSchedule; -use aqpbm_harness::bench_external::{run_lane, CellOutcome, LaneSetup}; -use aqpbm_harness::conformance::{self, ConformanceCheck}; -use aqpbm_harness::data::{workload_chunks, DataRecord, SeriesLabels}; -use aqpbm_harness::engine::{ +use aqpbm_core::harness::arm_run::ArmSchedule; +use aqpbm_core::harness::bench_external::{run_lane, CellOutcome, LaneSetup}; +use aqpbm_core::harness::conformance::{self, ConformanceCheck}; +use aqpbm_core::harness::data::{workload_chunks, DataRecord, SeriesLabels}; +use aqpbm_core::harness::engine::{ ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, QueryLanguage, QueryText, WorkloadQuery, WorkloadSpec, }; -use aqpbm_harness::protocol::{LoadProbe, MeasurementClocks, OsCpuClock, Protocol, ProtocolError}; -use aqpbm_harness::records::ArmResult; -use aqpbm_harness::rundir::RunDir; +use aqpbm_core::harness::protocol::{ + LoadProbe, MeasurementClocks, OsCpuClock, Protocol, ProtocolError, +}; +use aqpbm_core::harness::records::ArmResult; +use aqpbm_core::harness::rundir::RunDir; +use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, UniformParameter}; use asap_devtools::asap_lane::AsapEngine; use asap_planner::{plan_installable, FrontendInput, PlanningModels, PlanningWorkload, UserInput}; @@ -174,7 +176,7 @@ fn run_asap_lane( run: &RunDir, warmup: u32, runs: u32, -) -> Vec { +) -> Vec { let protocol = Protocol { warmup_runs: warmup, measured_runs: runs, @@ -508,7 +510,7 @@ fn asap_run_runs_the_asap_lane_over_a_run_dir() { "{}", String::from_utf8_lossy(&output.stderr) ); - let cell: aqpbm_harness::bench_external::LaneCell = serde_json::from_str( + let cell: aqpbm_core::harness::bench_external::LaneCell = serde_json::from_str( &std::fs::read_to_string(run.path().join("results/asap/w__exact.cell.json")).unwrap(), ) .unwrap(); From b56669894e379d68ac559009874a03c9c26c4433 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:56:39 +0000 Subject: [PATCH 6/7] test: the ASAP lane under the plan file the Planner's executor check no longer gates asap_planner --plan-file now writes every selected plan; a plan the Planner's executor cannot run comes with a stderr note. The lane tests write plan files without --executor-check, and the absent_over_time case checks the note, then the lane's not executable cell. --- crates/devtools/tests/asap_lane.rs | 39 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/crates/devtools/tests/asap_lane.rs b/crates/devtools/tests/asap_lane.rs index 836ba302e..897be5404 100644 --- a/crates/devtools/tests/asap_lane.rs +++ b/crates/devtools/tests/asap_lane.rs @@ -77,26 +77,22 @@ fn planning_workload( ) } -/// Writes `plan_file` with `asap_planner --plan-file` for `planning`, checked -/// as `executor_check` says; whether the planner wrote it. -fn write_plan_file(planning: &str, plan_file: &Path, executor_check: &str) -> bool { +/// Writes `plan_file` with `asap_planner --plan-file` for `planning`, which +/// must select a plan; returns the planner's standard error, which notes +/// when the Planner's executor cannot run the plan. +fn write_plan_file(planning: &str, plan_file: &Path) -> String { let workload_file = plan_file.with_extension("planning.json"); std::fs::write(&workload_file, planning).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_asap_planner")) .args(["--workload", workload_file.to_str().unwrap()]) .args(["--run-until", "stage3", "--format", "json"]) - .args(["--executor-check", executor_check]) .args(["--plan-file", plan_file.to_str().unwrap()]) .output() .unwrap(); - let written = plan_file.exists(); - assert_eq!( - output.status.success(), - written, - "{}", - String::from_utf8_lossy(&output.stderr) - ); - written + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(output.status.success(), "{stderr}"); + assert!(plan_file.exists(), "{stderr}"); + stderr } /// The query of workload `w`: a 5-minute window, so a plan built at @@ -332,7 +328,7 @@ fn the_asap_lane_runs_a_planned_sum_over_time_and_answers_exactly() { (&at_rest, r#""at_rest""#), ] { let planning = planning_workload(SUM_QUERY, "m", 1_000, 1_000, arrival); - assert!(write_plan_file(&planning, plan_file, "compile")); + write_plan_file(&planning, plan_file); } let run = run_dir( &temp, @@ -391,9 +387,9 @@ fn the_asap_lane_runs_a_planned_sum_over_time_and_answers_exactly() { } /// `absent_over_time(m[10s])`: Stages 1-3 plan it, but the Planner's -/// executor cannot compile it. The plan file written for a runtime other -/// than the executor (`--executor-check skip`) is a "not executable" cell -/// on the ASAP lane, so the other lanes still run that plan. +/// executor cannot compile it. `asap_planner` still writes the plan file +/// (with a note), and on the ASAP lane it is a "not executable" cell, so +/// the other lanes still run that plan. #[test] fn a_plan_the_executor_cannot_compile_is_a_not_executable_cell() { const QUERY: &str = "absent_over_time(m[10s])"; @@ -402,8 +398,11 @@ fn a_plan_the_executor_cannot_compile_is_a_not_executable_cell() { let plan_file = temp.0.join("absent.plan.json"); // Precondition: an executor-only gap. Once the executor runs // `absent_over_time`, pick another query it plans but cannot compile. - assert!(!write_plan_file(&planning, &plan_file, "compile")); - assert!(write_plan_file(&planning, &plan_file, "skip")); + let stderr = write_plan_file(&planning, &plan_file); + assert!( + stderr.contains("note: the Planner's executor cannot run this plan: "), + "{stderr}" + ); let workload = harness_workload("w", QUERY, 10_000); let run = run_dir(&temp, &[&workload], &[("exact", "w", plan_file)]); let cells = run_asap_lane(&run, 0, 1); @@ -427,7 +426,7 @@ fn a_plan_for_other_queries_is_refused_naming_the_query() { let temp = TempDir::new("refusals"); let planning = planning_workload(SUM_QUERY, "m", 2, 1_000, r#""at_rest""#); let plan_file = temp.0.join("sum.plan.json"); - assert!(write_plan_file(&planning, &plan_file, "compile")); + write_plan_file(&planning, &plan_file); let arm = |options: serde_json::Value| ArmSpec { name: "exact".into(), raw_data_retained: true, @@ -493,7 +492,7 @@ fn asap_run_runs_the_asap_lane_over_a_run_dir() { let workload = harness_workload("w", SUM_QUERY, 300_000); let plan_file = temp.0.join("sum.plan.json"); let planning = planning_workload(SUM_QUERY, "m", 2, 1_000, r#""at_rest""#); - assert!(write_plan_file(&planning, &plan_file, "compile")); + write_plan_file(&planning, &plan_file); let run = run_dir(&temp, &[&workload], &[("exact", "w", plan_file)]); let asap_run = |lane: &str| { Command::new(env!("CARGO_BIN_EXE_asap_run")) From 172f63a403778b08165a06b1c4674a79bbf81c04 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:25:02 +0000 Subject: [PATCH 7/7] test: run the ASAP lane's conformance check in its own test binary The check compares thread and process CPU; other tests running in the same process added process CPU the lane did not spend, so it failed whenever it ran beside them. --- crates/devtools/Cargo.toml | 5 + crates/devtools/tests/asap_lane.rs | 122 +----------------- .../devtools/tests/asap_lane_conformance.rs | 92 +++++++++++++ crates/devtools/tests/asap_lane_support.rs | 45 +++++++ 4 files changed, 146 insertions(+), 118 deletions(-) create mode 100644 crates/devtools/tests/asap_lane_conformance.rs create mode 100644 crates/devtools/tests/asap_lane_support.rs diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 931ac0c74..057e0bd9c 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -48,6 +48,11 @@ asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } name = "devtools" path = "tests/main.rs" +# Its own process: the check compares thread and process CPU. +[[test]] +name = "asap_lane_conformance" +path = "tests/asap_lane_conformance.rs" + # The nightly corpus-snapshots workflow runs this binary by name. [[test]] name = "corpus_snapshots" diff --git a/crates/devtools/tests/asap_lane.rs b/crates/devtools/tests/asap_lane.rs index 897be5404..c217218fa 100644 --- a/crates/devtools/tests/asap_lane.rs +++ b/crates/devtools/tests/asap_lane.rs @@ -8,11 +8,9 @@ use std::process::Command; use aqpbm_core::harness::arm_run::ArmSchedule; use aqpbm_core::harness::bench_external::{run_lane, CellOutcome, LaneSetup}; -use aqpbm_core::harness::conformance::{self, ConformanceCheck}; use aqpbm_core::harness::data::{workload_chunks, DataRecord, SeriesLabels}; use aqpbm_core::harness::engine::{ - ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, - QueryLanguage, QueryText, WorkloadQuery, WorkloadSpec, + ArmSpec, EngineError, EngineUnderTest, QueryLanguage, QueryText, WorkloadQuery, WorkloadSpec, }; use aqpbm_core::harness::protocol::{ LoadProbe, MeasurementClocks, OsCpuClock, Protocol, ProtocolError, @@ -21,26 +19,10 @@ use aqpbm_core::harness::records::ArmResult; use aqpbm_core::harness::rundir::RunDir; use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, UniformParameter}; use asap_devtools::asap_lane::AsapEngine; -use asap_planner::{plan_installable, FrontendInput, PlanningModels, PlanningWorkload, UserInput}; -/// A directory under the build's test temp directory, removed when dropped. -struct TempDir(PathBuf); - -impl TempDir { - fn new(test: &str) -> Self { - let path = PathBuf::from(env!("CARGO_TARGET_TMPDIR")) - .join(format!("asap-lane-{}-{test}", std::process::id())); - let _ = std::fs::remove_dir_all(&path); - std::fs::create_dir_all(&path).unwrap(); - TempDir(path) - } -} - -impl Drop for TempDir { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.0); - } -} +#[path = "asap_lane_support.rs"] +mod support; +use support::{planning_workload, TempDir}; /// Every core idle, so a test never waits on the machine's load. struct IdleProbe; @@ -54,29 +36,6 @@ impl LoadProbe for IdleProbe { } } -/// The planning workload of one exact PromQL `query` evaluated every 10 s at -/// phase 0 over gauge `metric` with `series_count` series sampled every -/// `sample_interval_ms`, arriving as `arrival` (`{"ingesting": ...}` or -/// `"at_rest"`). -fn planning_workload( - query: &str, - metric: &str, - series_count: u64, - sample_interval_ms: u64, - arrival: &str, -) -> String { - format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": {query:?}, - "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, - "requirements": {{"accuracy": {{"max_error": 0.0, "failure_probability": 0.0}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {arrival}, - "sources": {{{metric:?}: {{"series_count": {series_count}, "metric_type": "gauge", - "sample_interval": {sample_interval_ms}}}}}}}}}"# - ) -} - /// Writes `plan_file` with `asap_planner --plan-file` for `planning`, which /// must select a plan; returns the planner's standard error, which notes /// when the Planner's executor cannot run the plan. @@ -238,79 +197,6 @@ fn arm_result(run: &RunDir, workload: &str, variant: &str) -> ArmResult { serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() } -/// The conformance kit's probe arm has no engine options, so this engine -/// hands `AsapEngine` the plan file of each probe workload, by name. -struct WithPlanFiles { - plan_files: BTreeMap, -} - -impl EngineUnderTest for WithPlanFiles { - fn description(&self) -> EngineDescription { - AsapEngine.description() - } - fn cpu_accounting(&self) -> CpuAccounting { - AsapEngine.cpu_accounting() - } - fn install( - &self, - workload: &WorkloadSpec, - arm: &ArmSpec, - ) -> Result, EngineError> { - let arm = ArmSpec { - engine_options: serde_json::json!({"plan_file": self.plan_files[&workload.name]}), - ..arm.clone() - }; - AsapEngine.install(workload, &arm) - } -} - -/// The harness's conformance kit passes on the ASAP lane: deterministic -/// answers, left-open windows (`count_over_time(probe[10s])` at 20 s counts -/// only the sample at 20 s), and the executor's work on the calling thread. -#[test] -fn conformance_kit_passes_on_the_asap_engine() { - let temp = TempDir::new("conformance"); - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); - let mut plan_files = BTreeMap::new(); - for workload in conformance::probe_workloads() { - let query = &workload.queries[0].texts[0]; - assert_eq!(query.language, QueryLanguage::PromQl); - let planning: PlanningWorkload = serde_json::from_str(&planning_workload( - &query.text, - "probe", - 1, - 10_000, - r#"{"ingesting": {"rows_per_second": 0.1}}"#, - )) - .unwrap(); - let (plan, _) = runtime - .block_on(plan_installable(UserInput::new( - &planning, - FrontendInput::Promql, - PlanningModels::builtin(), - ))) - .unwrap(); - let path = temp.0.join(format!("{}.plan.json", workload.name)); - std::fs::write(&path, serde_json::to_string(&plan).unwrap()).unwrap(); - plan_files.insert(workload.name.clone(), path); - } - let mut engine = WithPlanFiles { plan_files }; - let report = conformance::check( - &mut engine, - &MeasurementClocks::with_calling_thread(&OsCpuClock), - ) - .unwrap_or_else(|failures| panic!("{failures:#?}")); - for check in [ - ConformanceCheck::DeterministicAnswers, - ConformanceCheck::LeftOpenWindow, - ConformanceCheck::WorkOnCallingThread, - ] { - assert!(report.passed.contains(&check), "{report:?}"); - } -} - /// `sum_over_time(m[5m])` every 10 s over 2 series, planned twice: with /// samples arriving (a chain of 10 s tumbling windows built at ingestion /// time) and at rest (raw samples scanned at query time). The ASAP lane diff --git a/crates/devtools/tests/asap_lane_conformance.rs b/crates/devtools/tests/asap_lane_conformance.rs new file mode 100644 index 000000000..645b44930 --- /dev/null +++ b/crates/devtools/tests/asap_lane_conformance.rs @@ -0,0 +1,92 @@ +//! The harness's conformance kit on the ASAP lane. It checks that the +//! executor's work runs on the calling thread by comparing thread and process +//! CPU, so it runs in its own test binary: other tests running in the same +//! process would add process CPU the lane did not spend. +use std::collections::BTreeMap; +use std::path::PathBuf; + +use aqpbm_core::harness::conformance::{self, ConformanceCheck}; +use aqpbm_core::harness::engine::{ + ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, + QueryLanguage, WorkloadSpec, +}; +use aqpbm_core::harness::protocol::{MeasurementClocks, OsCpuClock}; +use asap_devtools::asap_lane::AsapEngine; +use asap_planner::{plan_installable, FrontendInput, PlanningModels, PlanningWorkload, UserInput}; + +#[path = "asap_lane_support.rs"] +mod support; +use support::{planning_workload, TempDir}; + +/// The conformance kit's probe arm has no engine options, so this engine +/// hands `AsapEngine` the plan file of each probe workload, by name. +struct WithPlanFiles { + plan_files: BTreeMap, +} + +impl EngineUnderTest for WithPlanFiles { + fn description(&self) -> EngineDescription { + AsapEngine.description() + } + fn cpu_accounting(&self) -> CpuAccounting { + AsapEngine.cpu_accounting() + } + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let arm = ArmSpec { + engine_options: serde_json::json!({"plan_file": self.plan_files[&workload.name]}), + ..arm.clone() + }; + AsapEngine.install(workload, &arm) + } +} + +/// The harness's conformance kit passes on the ASAP lane: deterministic +/// answers, left-open windows (`count_over_time(probe[10s])` at 20 s counts +/// only the sample at 20 s), and the executor's work on the calling thread. +#[test] +fn conformance_kit_passes_on_the_asap_engine() { + let temp = TempDir::new("conformance"); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let mut plan_files = BTreeMap::new(); + for workload in conformance::probe_workloads() { + let query = &workload.queries[0].texts[0]; + assert_eq!(query.language, QueryLanguage::PromQl); + let planning: PlanningWorkload = serde_json::from_str(&planning_workload( + &query.text, + "probe", + 1, + 10_000, + r#"{"ingesting": {"rows_per_second": 0.1}}"#, + )) + .unwrap(); + let (plan, _) = runtime + .block_on(plan_installable(UserInput::new( + &planning, + FrontendInput::Promql, + PlanningModels::builtin(), + ))) + .unwrap(); + let path = temp.0.join(format!("{}.plan.json", workload.name)); + std::fs::write(&path, serde_json::to_string(&plan).unwrap()).unwrap(); + plan_files.insert(workload.name.clone(), path); + } + let mut engine = WithPlanFiles { plan_files }; + let report = conformance::check( + &mut engine, + &MeasurementClocks::with_calling_thread(&OsCpuClock), + ) + .unwrap_or_else(|failures| panic!("{failures:#?}")); + for check in [ + ConformanceCheck::DeterministicAnswers, + ConformanceCheck::LeftOpenWindow, + ConformanceCheck::WorkOnCallingThread, + ] { + assert!(report.passed.contains(&check), "{report:?}"); + } +} diff --git a/crates/devtools/tests/asap_lane_support.rs b/crates/devtools/tests/asap_lane_support.rs new file mode 100644 index 000000000..ad6f2faba --- /dev/null +++ b/crates/devtools/tests/asap_lane_support.rs @@ -0,0 +1,45 @@ +//! Helpers shared by the ASAP lane tests (`asap_lane.rs`, in the `devtools` +//! test binary) and the conformance test, which runs in its own binary. +use std::path::PathBuf; + +/// A directory under the build's test temp directory, removed when dropped. +pub struct TempDir(pub PathBuf); + +impl TempDir { + pub fn new(test: &str) -> Self { + let path = PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(format!("asap-lane-{}-{test}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + TempDir(path) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// The planning workload of one exact PromQL `query` evaluated every 10 s at +/// phase 0 over gauge `metric` with `series_count` series sampled every +/// `sample_interval_ms`, arriving as `arrival` (`{"ingesting": ...}` or +/// `"at_rest"`). +pub fn planning_workload( + query: &str, + metric: &str, + series_count: u64, + sample_interval_ms: u64, + arrival: &str, +) -> String { + format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ + "query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {{"max_error": 0.0, "failure_probability": 0.0}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {arrival}, + "sources": {{{metric:?}: {{"series_count": {series_count}, "metric_type": "gauge", + "sample_interval": {sample_interval_ms}}}}}}}}}"# + ) +}