From dcc5f2fa5c8438e91cb7d6fb53d0d3ea9a561b69 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:43:00 +0000 Subject: [PATCH] test(devtools): one corpus registry of every in-repo workload and builders that plan through the production frontends and Stage 0; snapshots read the registry (#1253 T1) --- crates/devtools/src/bin/asap_planner.rs | 26 +- crates/devtools/src/corpus_registry.rs | 671 ++++++++++++++++++ crates/devtools/src/lib.rs | 3 + crates/devtools/src/planner_fuzz/mod.rs | 41 +- crates/devtools/src/planning_stages.rs | 42 +- crates/devtools/tests/corpus_snapshots.rs | 136 ++-- .../corpus_snapshots/o11y_bench/corpus.json | 1 - crates/devtools/tests/planner_fuzz.rs | 26 +- .../devtools/tests/sketch_plan_snapshots.rs | 116 +-- .../integration-tests/tests/milp_objective.rs | 90 +-- docs/develop_docs/planner-snapshots.md | 5 +- docs/develop_docs/testing.md | 13 +- 12 files changed, 924 insertions(+), 246 deletions(-) create mode 100644 crates/devtools/src/corpus_registry.rs diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index c7cf44720..1e4462bc9 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -109,8 +109,8 @@ use asap_accuracy::{accepted_metrics, AccuracyEvidence}; use asap_cost::{CostInputs, PlanCost}; use asap_devtools::candidate_labels::operator_label; use asap_devtools::planning_stages::{ - lower_promql_roots, lower_sql_roots, run_stages_until, sql_catalog, PlanningStagesError, - PlanningStagesRun, RunUntil, + lower_workload, planning_models, run_stages_until, sql_catalog, with_executor_summaries, + PlanningStagesError, PlanningStagesRun, RunUntil, }; use asap_devtools::planning_stages_text::{ column_lineage_text, edges_per_input, logical_edges_into, physical_edge_rejection, @@ -118,7 +118,7 @@ use asap_devtools::planning_stages_text::{ }; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; +use asap_planner::{installable_plan_of, SelectionError}; use asap_stage3_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::flat::{flatten, FlatDag}; @@ -224,15 +224,12 @@ fn run(args: Vec) -> Result<(), Failure> { } let (catalog, table_names) = sql_catalog(&tables)?; let (described, roots, entries, data) = lowered_together(&workloads, &catalog, &table_names)?; - // Plan with the executor's summary families when the deployment declares none. - if deployment_capabilities.supported_summaries.is_none() { - deployment_capabilities.supported_summaries = - Some(asap_executor::supported_summary_families()); - } - let models = PlanningModels::builtin() - .with_cost_inputs(&cost_inputs) - .with_deployment_capabilities(&deployment_capabilities) - .with_ignore_accuracy_guarantees(ignore_accuracy_guarantees); + let deployment_capabilities = with_executor_summaries(deployment_capabilities); + let models = planning_models( + &deployment_capabilities, + &cost_inputs, + ignore_accuracy_guarantees, + ); let evidence = models.evidence; let result = run_stages_until(roots.clone(), &entries, &data, models, run_until); // The selected plan's cost, as `plan_installable` reports it. @@ -498,10 +495,7 @@ type Lowered = ( /// `workload` lowered by its language's frontend (SQL over `catalog`); its /// data workload is empty when it declares none. fn lowered(workload: &PlanningWorkload, catalog: &SqlCatalog) -> Result { - let roots = match workload.query_workload.language { - QueryLanguage::PromQL => lower_promql_roots(workload)?, - _ => lower_sql_roots(workload, catalog)?, - }; + let roots = lower_workload(workload, catalog)?; let entries = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap_or_default(); Ok((workload_queries(workload), roots, entries, data)) diff --git a/crates/devtools/src/corpus_registry.rs b/crates/devtools/src/corpus_registry.rs new file mode 100644 index 000000000..8d76320d5 --- /dev/null +++ b/crates/devtools/src/corpus_registry.rs @@ -0,0 +1,671 @@ +//! The one list of in-repo workloads that the contracts suite, the planner +//! snapshots and the oracles read (`docs/develop_docs/testing.md`), and the +//! builders that plan a workload only through the production path: the +//! query-language frontends, then Stage 0 to Stage 3 as `asap_planner` +//! runs them. +//! +//! [`corpus_workloads`] reads every source from the repository, so a +//! workload added to one of them is planned by every suite that iterates +//! the registry: +//! +//! | Id prefix | Source | +//! |---|---| +//! | `o11y_bench/` | `crates/devtools/tests/corpus_snapshots/o11y_bench/corpus.json`, one workload per query | +//! | `sketch_plans/` | `crates/devtools/tests/sketch_plan_snapshots/manifest.json` (MVP and raw-retained deployments) | +//! | `planner_layering/` | `tools/dag-viewer/workloads/planner-layering/` | +//! | `memo_examples/` | `tools/dag-viewer/memo-examples/{promql,sql}/` | +//! | `random_mixed/` | the sets of `tools/dag-viewer/workloads/index.json` | +//! | `fuzz_seeds/` | `crates/devtools/tests/planner_fuzz_seeds/` (planned as `planner_fuzz` plans: accuracy not enforced) | +//! +//! No ASAPWorkloads workload files exist in this repository (its query +//! catalog appears only as `planner_fuzz` generator patterns), so there is +//! no source for them. + +use std::path::{Path, PathBuf}; + +use asap_cost::CostInputs; +use asap_planner::PlanSearchSettings; +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::workload::{ + DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryWorkloadEntry, SourceFacts, +}; +use serde_json::{json, Value}; + +use crate::planner_fuzz::{read_seed_file, FuzzWorkload}; +use crate::planning_stages::{ + lower_workload, planning_models, run_search_over, run_stages_until, sql_catalog, + with_executor_summaries, PlanningStagesRun, RunUntil, +}; + +/// Every source's id prefix, in [`corpus_workloads`] order: a workload's id +/// is `/`. +pub const SOURCES: [&str; 6] = [ + "o11y_bench", + "sketch_plans", + "planner_layering", + "memo_examples", + "random_mixed", + "fuzz_seeds", +]; + +/// One registry workload with the deployment it is planned for: what +/// `asap_planner` is given for it. +#[derive(Debug, Clone)] +pub struct CorpusWorkload { + /// `/`, unique in the registry, e.g. `o11y_bench/q21`. + pub id: String, + /// The workload as a user writes it (`asap_planner --workload`). + pub workload: PlanningWorkload, + /// A SQL workload's tables, each an `asap_planner --sql-catalog-table` + /// JSON object; empty for PromQL. + pub sql_tables: Vec, + /// What the deployment's executor runs and keeps + /// (`--deployment-capabilities`); `supported_summaries: None` plans with + /// the executor's summary families, as `asap_planner` does. + pub deployment_capabilities: DeploymentCapabilities, + /// The deployment's cost model and unit costs (`--cost-inputs`). + pub cost_inputs: CostInputs, + /// Plan with accuracy not enforced (`--ignore-accuracy-guarantees`). + pub ignore_accuracy_guarantees: bool, +} + +/// A workload planned through the production path, with what costing its +/// plan needs besides the cost inputs it was planned with. +#[derive(Debug)] +pub struct PlannedWorkload { + /// One entry per query, in workload order (`QueryWorkload::entries`). + pub entries: Vec, + /// The data workload (empty when the workload declares none). + pub data: DataWorkload, + /// Every stage's output up to the one planned until. + pub stages: PlanningStagesRun, +} + +/// `workload` planned as `asap_planner --run-until ` plans it, with +/// the binary's lowering (`planning_stages::lower_workload`, SQL over +/// `sql_tables`) and models (`planning_stages::planning_models` over +/// `with_executor_summaries`): validated, lowered, then Stage 0 up to +/// `until`. Until Stage 3 it runs the facade's `run_planning` (with +/// `inspect`, so every stage's output is kept), as `planner_fuzz` plans. +/// A stage that fails after Pass 1 is reported in the run, as +/// `asap_planner` reports it; validation, lowering and Pass 1 errors are +/// returned. +pub fn plan_workload( + workload: &PlanningWorkload, + sql_tables: &[Value], + deployment_capabilities: &DeploymentCapabilities, + cost_inputs: &CostInputs, + ignore_accuracy_guarantees: bool, + until: RunUntil, +) -> Result { + workload.validate().map_err(|e| e.to_string())?; + let tables: Vec = sql_tables.iter().map(Value::to_string).collect(); + let roots = lower_workload(workload, &sql_catalog(&tables)?.0)?; + let entries: Vec = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let capabilities = with_executor_summaries(deployment_capabilities.clone()); + let models = planning_models(&capabilities, cost_inputs, ignore_accuracy_guarantees); + let stages = match until { + RunUntil::Stage3 => run_search_over( + roots, + &entries, + &data, + models, + PlanSearchSettings { inspect: true }, + )?, + _ => run_stages_until(roots, &entries, &data, models, until)?, + }; + Ok(PlannedWorkload { + entries, + data, + stages, + }) +} + +impl CorpusWorkload { + /// This workload planned for its deployment up to `until` ([`plan_workload`]). + pub fn plan(&self, until: RunUntil) -> Result { + plan_workload( + &self.workload, + &self.sql_tables, + &self.deployment_capabilities, + &self.cost_inputs, + self.ignore_accuracy_guarantees, + until, + ) + } + + /// The `asap_planner` arguments for this workload, its input files + /// written to `directory`: `--workload`, each `--sql-catalog-table`, + /// `--deployment-capabilities`, `--cost-inputs` and + /// `--ignore-accuracy-guarantees` when set. + pub fn asap_planner_args(&self, directory: &Path) -> Result, String> { + let file_name = self.id.replace('/', "__"); + let write = |suffix: &str, value: String| -> Result { + let path = directory.join(format!("{file_name}.{suffix}.json")); + std::fs::write(&path, value).map_err(|e| format!("{}: {e}", path.display()))?; + Ok(path.display().to_string()) + }; + fn to_json(value: &impl serde::Serialize) -> Result { + serde_json::to_string(value).map_err(|e| e.to_string()) + } + let mut args = vec![ + "--workload".to_owned(), + write("workload", to_json(&self.workload)?)?, + ]; + for table in &self.sql_tables { + args.extend(["--sql-catalog-table".to_owned(), table.to_string()]); + } + args.extend([ + "--deployment-capabilities".to_owned(), + write("capabilities", to_json(&self.deployment_capabilities)?)?, + "--cost-inputs".to_owned(), + write("cost_inputs", to_json(&self.cost_inputs)?)?, + ]); + if self.ignore_accuracy_guarantees { + args.push("--ignore-accuracy-guarantees".to_owned()); + } + Ok(args) + } +} + +/// The repository root. +fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn read_json(path: &Path) -> Result { + let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display())) +} + +fn parse(value: Value, what: &str) -> Result { + serde_json::from_value(value).map_err(|e| format!("{what}: {e}")) +} + +/// `object[key]` as a string, or an error naming `what` and the key. +fn string_field<'a>(object: &'a Value, key: &str, what: &str) -> Result<&'a str, String> { + object[key] + .as_str() + .ok_or_else(|| format!("{what}: no {key}")) +} + +/// Whether a manifest's or index's `accuracy_guarantees` (`"enforced"`, +/// the default, or `"ignored"`) ignores them. +fn accuracy_guarantees_ignored(value: &Value, what: &str) -> Result { + match value.as_str() { + None | Some("enforced") => Ok(false), + Some("ignored") => Ok(true), + Some(other) => Err(format!("{what}: accuracy_guarantees {other}")), + } +} + +/// A workload of the default deployment (raw data retained, the +/// executor's summary families, cost model 1 with illustrative unit costs), +/// accuracy enforced. +fn default_deployment( + id: String, + workload: PlanningWorkload, + sql_tables: Vec, +) -> CorpusWorkload { + CorpusWorkload { + id, + workload, + sql_tables, + deployment_capabilities: DeploymentCapabilities::default(), + cost_inputs: CostInputs::default(), + ignore_accuracy_guarantees: false, + } +} + +/// Every registry workload, in source order (see the module docs). Errors +/// name the file that does not parse. +pub fn corpus_workloads() -> Result, String> { + let mut workloads = o11y_bench()?; + workloads.extend(sketch_plans()?); + workloads.extend(planner_layering()?); + workloads.extend(memo_examples()?); + workloads.extend(random_mixed()?); + workloads.extend(fuzz_seeds()?); + Ok(workloads) +} + +/// The o11y benchmark corpus: one repeating query per workload under the +/// corpus's shared entry and data, each id `o11y_bench/`; a query +/// with a `skip_reason` is left out. +pub fn o11y_bench() -> Result, String> { + let path = + repository_root().join("crates/devtools/tests/corpus_snapshots/o11y_bench/corpus.json"); + let corpus = read_json(&path)?; + let queries = corpus["queries"] + .as_array() + .ok_or("corpus.json: no queries")?; + queries + .iter() + .filter(|query| query.get("skip_reason").is_none()) + .map(|query| { + let mut entry = corpus["query_workload"]["repeating_query"].clone(); + entry["query"] = query["query"].clone(); + let workload = json!({ + "query_workload": { + "language": corpus["query_workload"]["language"], + "repeating_queries": [entry], + }, + "data_workload": corpus["data_workload"], + }); + let id = string_field(query, "id", "corpus.json: a query")?; + Ok(default_deployment( + format!("o11y_bench/{id}"), + parse(workload, id)?, + Vec::new(), + )) + }) + .collect() +} + +/// The sketch plan snapshots' workloads (`manifest.json`), each with its +/// deployment's capabilities and cost inputs, ids `sketch_plans/`. +pub fn sketch_plans() -> Result, String> { + let root = repository_root(); + let manifest = + read_json(&root.join("crates/devtools/tests/sketch_plan_snapshots/manifest.json"))?; + let entries = manifest["workloads"] + .as_array() + .ok_or("manifest.json: no workloads")?; + entries + .iter() + .map(|entry| { + let id = string_field(entry, "id", "manifest.json: an entry")?; + let deployment_name = string_field(entry, "deployment", id)?; + let deployment: Vec = parse( + manifest["deployments"] + .get(deployment_name) + .ok_or_else(|| format!("{id}: no deployment {deployment_name}"))? + .clone(), + &format!("{id}: deployment"), + )?; + let mut workload = default_deployment( + format!("sketch_plans/{id}"), + parse( + read_json(&root.join(string_field(entry, "workload", id)?))?, + id, + )?, + match entry.get("tables") { + Some(_) => parse( + read_json(&root.join(string_field(entry, "tables", id)?))?, + id, + )?, + None => Vec::new(), + }, + ); + for pair in deployment.chunks(2) { + match pair { + [flag, file] if flag == "--deployment-capabilities" => { + workload.deployment_capabilities = + parse(read_json(&root.join(file))?, file)? + } + [flag, file] if flag == "--cost-inputs" => { + workload.cost_inputs = parse(read_json(&root.join(file))?, file)? + } + other => return Err(format!("{id}: unknown deployment argument {other:?}")), + } + } + workload.ignore_accuracy_guarantees = + accuracy_guarantees_ignored(&entry["accuracy_guarantees"], id)?; + Ok(workload) + }) + .collect() +} + +/// The `*.json` workloads of `directory` (except `tables.json`), sorted by +/// name, with the directory's `tables.json` when it has one. +fn workload_files(directory: &Path) -> Result)>, String> { + let tables: Vec = match directory.join("tables.json").is_file() { + true => parse(read_json(&directory.join("tables.json"))?, "tables.json")?, + false => Vec::new(), + }; + let mut files: Vec = std::fs::read_dir(directory) + .map_err(|e| format!("{}: {e}", directory.display()))? + .map(|entry| entry.map(|entry| entry.path()).map_err(|e| e.to_string())) + .collect::>()?; + files.retain(|path| { + path.extension().is_some_and(|ext| ext == "json") && !path.ends_with("tables.json") + }); + files.sort(); + files + .into_iter() + .map(|path| { + let name = path.file_stem().unwrap().to_string_lossy().into_owned(); + let workload: PlanningWorkload = parse(read_json(&path)?, &path.display().to_string())?; + let tables = match workload.query_workload.language { + QueryLanguage::PromQL => Vec::new(), + _ => tables.clone(), + }; + Ok((name, workload, tables)) + }) + .collect() +} + +/// The #509 planner-layering examples as the viewer plans them (default +/// deployment, accuracy enforced), ids `planner_layering/`. +fn planner_layering() -> Result, String> { + let directory = repository_root().join("tools/dag-viewer/workloads/planner-layering"); + Ok(workload_files(&directory)? + .into_iter() + .map(|(name, workload, tables)| { + default_deployment(format!("planner_layering/{name}"), workload, tables) + }) + .collect()) +} + +/// The viewer's memo examples (default deployment, accuracy enforced), ids +/// `memo_examples//`. +fn memo_examples() -> Result, String> { + let mut workloads = Vec::new(); + for language in ["promql", "sql"] { + let directory = repository_root() + .join("tools/dag-viewer/memo-examples") + .join(language); + for (name, workload, tables) in workload_files(&directory)? { + workloads.push(default_deployment( + format!("memo_examples/{language}/{name}"), + workload, + tables, + )); + } + } + Ok(workloads) +} + +/// The viewer's random mixed workload sets (`index.json`), each workload +/// with its set's deployment capabilities, cost model and accuracy, ids +/// `random_mixed//`. +fn random_mixed() -> Result, String> { + let directory = repository_root().join("tools/dag-viewer/workloads"); + let index = read_json(&directory.join("index.json"))?; + let mut workloads = Vec::new(); + for set in index["sets"].as_array().ok_or("index.json: no sets")? { + let set_id = string_field(set, "id", "index.json: a set")?; + let capabilities: DeploymentCapabilities = + parse(set["deployment_capabilities"].clone(), set_id)?; + let cost_inputs: CostInputs = match set["cost_model"].as_str() { + Some(name) => parse( + read_json( + &repository_root().join(format!("tools/dag-viewer/cost-inputs/{name}.json")), + )?, + name, + )?, + None => CostInputs::default(), + }; + let ignore = accuracy_guarantees_ignored(&set["accuracy_guarantees"], set_id)?; + for workload in set["workloads"] + .as_array() + .ok_or("index.json: no workloads")? + { + let id = string_field(workload, "id", set_id)?; + let file = directory.join(string_field(workload, "file", id)?); + workloads.push(CorpusWorkload { + id: format!("random_mixed/{set_id}/{id}"), + workload: parse(read_json(&file)?, &file.display().to_string())?, + sql_tables: Vec::new(), + deployment_capabilities: capabilities.clone(), + cost_inputs: cost_inputs.clone(), + ignore_accuracy_guarantees: ignore, + }); + } + } + Ok(workloads) +} + +/// The `planner_fuzz` seed workloads as the tester reads them, with their +/// registry ids `fuzz_seeds/` (e.g. `fuzz_seeds/issue902`). +pub fn fuzz_seed_workloads() -> Result, String> { + let directory = repository_root().join("crates/devtools/tests/planner_fuzz_seeds"); + let mut files: Vec = std::fs::read_dir(&directory) + .map_err(|e| format!("{}: {e}", directory.display()))? + .map(|entry| entry.map(|entry| entry.path()).map_err(|e| e.to_string())) + .collect::>()?; + files.retain(|path| path.extension().is_some_and(|ext| ext == "json")); + files.sort(); + files + .into_iter() + .map(|path| { + let name = path.file_name().unwrap().to_string_lossy(); + let name = name.trim_end_matches(".json").trim_end_matches(".workload"); + Ok((format!("fuzz_seeds/{name}"), read_seed_file(&path)?)) + }) + .collect() +} + +/// The `planner_fuzz` seeds, planned as the tester plans them: accuracy not +/// enforced, default cost inputs, the seed's deployment. +fn fuzz_seeds() -> Result, String> { + Ok(fuzz_seed_workloads()? + .into_iter() + .map(|(id, seed)| CorpusWorkload { + id, + workload: seed.planning_workload(), + sql_tables: seed.tables.clone(), + deployment_capabilities: seed.deployment(), + cost_inputs: CostInputs::default(), + ignore_accuracy_guarantees: true, + }) + .collect()) +} + +/// The ingesting arrival at the sum of `data`'s sources' own rates +/// (`series_count` per `sample_interval`), when every source declares one +/// and the sum is positive. +pub fn arrival_from_sources(data: &DataWorkload) -> Option { + let rows_per_second: f64 = data + .sources + .values() + .map(|facts| match (facts.series_count, facts.sample_interval) { + (Some(series), Some(interval)) if interval.0 > 0 => { + Some(series as f64 * 1_000.0 / interval.0 as f64) + } + _ => None, + }) + .sum::>()?; + (rows_per_second > 0.0).then_some(DataArrival::Ingesting { + rows_per_second, + max_out_of_order_ms: None, + }) +} + +/// A PromQL workload of `queries`, each repeating every `interval_ms` at +/// phase 0 with accuracy `accuracy`, over `sources` (metric → its facts), +/// arriving at the sum of the sources' rates when each declares +/// `series_count` and `sample_interval` ([`arrival_from_sources`]). For +/// rule and oracle tests that plan a small workload through the production +/// path ([`plan_workload`]). +pub fn repeating_promql_workload( + queries: &[&str], + interval_ms: u64, + accuracy: AccuracyTarget, + sources: &[(&str, SourceFacts)], +) -> PlanningWorkload { + let entries: Vec = queries + .iter() + .map(|query| { + json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": interval_ms, "evaluation_phase": 0}}, + "requirements": {"accuracy": accuracy}, + "predictability": {"predictable": {"known_at": null}}, + }) + }) + .collect(); + let mut workload: PlanningWorkload = parse( + json!({ + "query_workload": {"language": "promql", "query_batch": null, "repeating_queries": entries}, + "data_workload": {"arrival": "unknown"}, + }), + "repeating_promql_workload", + ) + .expect("a repeating PromQL workload deserializes"); + let data = workload.data_workload.as_mut().expect("declared above"); + data.sources = sources + .iter() + .map(|(name, facts)| (name.to_string(), facts.clone())) + .collect(); + if let Some(arrival) = arrival_from_sources(data) { + data.arrival = arrival; + } + workload +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_planner::SelectionError; + use asap_types::ir::QueryRoot; + use asap_types::workload::DurationMs; + + fn facts(series_count: u64, sample_interval_ms: u64) -> SourceFacts { + SourceFacts { + series_count: Some(series_count), + sample_interval: Some(DurationMs(sample_interval_ms)), + ..SourceFacts::default() + } + } + + /// Every source loads with the workload count of its files today, ids + /// are unique, and named workloads of each source are present. The + /// o11y count is its committed snapshots (`.json`), a listing the + /// registry does not read. + #[test] + fn every_source_loads_with_unique_ids() { + let workloads = corpus_workloads().unwrap(); + let mut ids: Vec<&str> = workloads.iter().map(|w| w.id.as_str()).collect(); + ids.sort(); + let count = ids.len(); + ids.dedup(); + assert_eq!(ids.len(), count, "duplicate ids"); + let o11y_snapshots = std::fs::read_dir( + repository_root().join("crates/devtools/tests/corpus_snapshots/o11y_bench"), + ) + .unwrap() + .filter(|entry| { + let name = entry.as_ref().unwrap().file_name(); + let name = name.to_string_lossy(); + name.ends_with(".json") && name != "corpus.json" + }) + .count(); + let expected = [ + ("o11y_bench", o11y_snapshots), + ("sketch_plans", 10), + ("planner_layering", 4), + ("memo_examples", 7), + ("random_mixed", 30), + ("fuzz_seeds", 3), + ]; + assert_eq!(expected.map(|(source, _)| source), SOURCES); + for (source, count) in expected { + let prefix = format!("{source}/"); + assert_eq!( + ids.iter().filter(|id| id.starts_with(&prefix)).count(), + count, + "{source}" + ); + } + for id in [ + "o11y_bench/q21", + "sketch_plans/shared-quantiles-1h-mvp", + "planner_layering/example2", + "memo_examples/sql/rollupheavy-t11-service-endpoint-5m", + "fuzz_seeds/issue902", + ] { + assert!(ids.contains(&id), "no {id}"); + } + } + + /// Two sources at 1,000 series every 1 s and 100 series every 10 s + /// arrive at 1,010 rows per second; each query repeats every 60 s. + #[test] + fn a_repeating_promql_workload_arrives_at_its_sources_rates() { + let workload = repeating_promql_workload( + &["sum(a)", "sum(b)"], + 60_000, + AccuracyTarget::exact(), + &[("a", facts(1000, 1000)), ("b", facts(100, 10_000))], + ); + assert_eq!( + workload.data_workload.unwrap().arrival.rows_per_second(), + Some(1_010.0) + ); + assert_eq!(workload.query_workload.entries().count(), 2); + } + + /// Planning until Pass 1 runs the frontends and Stage 0, keeps the + /// series identity the PromQL lowering adds on every `Scan`, and leaves + /// the later stages empty. + #[test] + fn planning_until_a_stage_leaves_the_later_ones_empty() { + let workload = repeating_promql_workload( + &["sum(rate(m[5m]))"], + 60_000, + AccuracyTarget::exact(), + &[("m", facts(10, 1000))], + ); + let planned = plan_workload( + &workload, + &[], + &DeploymentCapabilities::default(), + &CostInputs::default(), + false, + RunUntil::Pass1, + ) + .unwrap(); + let run = &planned.stages.run; + assert!(run.pass1_memo.is_some()); + assert!(run.logical_asap_candidates.is_none()); + assert!(run.physical_asap_candidates.is_none()); + assert!(matches!(run.selection, Err(SelectionError::NotRun))); + let QueryRoot::Operator(root) = &planned.stages.stage0_query_language_frontends[0] else { + panic!("an operator root") + }; + let scan_schemas: Vec<_> = asap_types::ir::OperatorNode::reachable(root) + .into_iter() + .filter(|node| { + matches!( + node.non_asap(), + Some(asap_types::ir::NonASAPOp::Scan { .. }) + ) + }) + .map(|node| node.schema.clone()) + .collect(); + assert!(!scan_schemas.is_empty(), "no Scan reached from the root"); + assert!( + scan_schemas.iter().all(|schema| schema + .fields + .iter() + .any(|field| field.name == asap_types::ir::schema::PROMQL_SERIES_IDENTITY)), + "{scan_schemas:?}" + ); + } + + /// A MetricsQL workload is an unsupported language here, not a SQL + /// parse error. + #[test] + fn a_language_without_a_frontend_here_is_reported_as_unsupported() { + let mut workload = repeating_promql_workload( + &["sum(m)"], + 60_000, + AccuracyTarget::exact(), + &[("m", facts(10, 1000))], + ); + workload.query_workload.language = QueryLanguage::MetricsQL; + let error = plan_workload( + &workload, + &[], + &DeploymentCapabilities::default(), + &CostInputs::default(), + false, + RunUntil::Stage0, + ) + .unwrap_err(); + assert!(error.contains("unsupported query language"), "{error}"); + } +} diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index 2b1e21f71..855a43e91 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -12,6 +12,8 @@ //! only) or [`asap_frontend_sql`] (DataFusion only) — so it never compiles the //! other's parser. //! +//! [`corpus_registry`] lists every in-repo workload the contracts, +//! snapshots and oracles plan, and plans one through the production path. //! [`planning_stages`] and [`planner_layering_examples`] hold the one //! pipeline and the #509 example workloads that `asap_planner` and the //! planner-layering acceptance tests share; [`candidate_labels`] names the @@ -23,6 +25,7 @@ pub mod candidate_labels; pub mod corpora; +pub mod corpus_registry; pub mod planner_fuzz; pub mod planner_layering_examples; pub mod planning_stages; diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index be9efa59a..2891164d3 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -47,7 +47,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use asap_cost::{plan_cost, CostInputs}; -use asap_planner::{PlanSearchSettings, PlanningModels}; +use asap_planner::PlanSearchSettings; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::flat::flatten; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId}; @@ -57,11 +57,12 @@ use asap_types::ir::schema::sketch::SketchAlgorithm; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot}; use asap_types::physical::{Lifetime, StorageMedium}; use asap_types::workload::{ - DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryWorkload, QueryWorkloadEntry, + DataWorkload, PlanningWorkload, QueryLanguage, QueryWorkload, QueryWorkloadEntry, RepeatingEntry, SqlDialect, }; -use crate::planning_stages::{lower_promql_roots, lower_sql_roots, run_search_over, sql_catalog}; +use crate::corpus_registry::arrival_from_sources; +use crate::planning_stages::{lower_workload, planning_models, run_search_over, sql_catalog}; use generate::QuerySpec; /// The language of a fuzz workload: one per workload, as one @@ -165,23 +166,10 @@ impl FuzzWorkload { } /// Sets the ingesting arrival rate to the sum of the sources' own rates - /// (`series_count` per `sample_interval`), when every source declares - /// one. + /// ([`arrival_from_sources`]), when every source declares one. pub fn set_arrival_from_sources(&mut self) { - let mut rows_per_second = 0.0; - for facts in self.data.sources.values() { - match (facts.series_count, facts.sample_interval) { - (Some(series), Some(interval)) if interval.0 > 0 => { - rows_per_second += series as f64 * 1_000.0 / interval.0 as f64 - } - _ => return, - } - } - if rows_per_second > 0.0 { - self.data.arrival = DataArrival::Ingesting { - rows_per_second, - max_out_of_order_ms: None, - }; + if let Some(arrival) = arrival_from_sources(&self.data) { + self.data.arrival = arrival; } } @@ -479,14 +467,8 @@ impl Planner { /// The frontends' roots of `workload`, one per query. pub fn lower(&self, workload: &FuzzWorkload) -> Result, String> { - let planning = workload.planning_workload(); - match workload.language { - Language::Promql => lower_promql_roots(&planning), - Language::Sql => { - let tables: Vec = workload.tables.iter().map(Value::to_string).collect(); - lower_sql_roots(&planning, &sql_catalog(&tables)?.0) - } - } + let tables: Vec = workload.tables.iter().map(Value::to_string).collect(); + lower_workload(&workload.planning_workload(), &sql_catalog(&tables)?.0) } /// `workload` planned by the configured search, with accuracy not @@ -508,10 +490,7 @@ impl Planner { let data = workload.data.clone(); let cost_inputs = self.cost_inputs(); let deployment = workload.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&cost_inputs) - .with_deployment_capabilities(&deployment) - .with_ignore_accuracy_guarantees(true); + let models = planning_models(&deployment, &cost_inputs, true); let run = match self.settings.search { SearchKind::CandidateConnections => run_search_over( roots.clone(), diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index d737742ab..1c7d5af9b 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -2,6 +2,7 @@ //! search for the planner-layering acceptance tests, and //! [`run_stages_until`] for `asap_planner`. +use asap_cost::CostInputs; use asap_frontend_sql::SqlCatalog; use asap_planner::{ run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectionError, @@ -11,11 +12,12 @@ use asap_stage1_pass1_per_computation_candidates::pass1_memo; use asap_stage1_pass2_reuse_candidates::generate_logical_asap_memo_candidates; use asap_stage2_physical_optimization::generate_physical_asap_memo_candidates; use asap_stage3_plan_selection::select_plan; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::QueryRoot; use asap_types::stages::Memo; -use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; +use asap_types::workload::{DataWorkload, PlanningWorkload, QueryLanguage, QueryWorkloadEntry}; use serde_json::Value; /// The last stage a run runs (`asap_planner --run-until`); the run's @@ -117,6 +119,44 @@ pub fn lower_sql_roots( .collect() } +/// `workload` lowered by its language's frontend (SQL over `catalog`), one +/// root per query in workload order, each PromQL series' identity as a +/// column. A language with no frontend here (MetricsQL, Elastic DSL) is an +/// error. +pub fn lower_workload( + workload: &PlanningWorkload, + catalog: &SqlCatalog, +) -> Result, String> { + match &workload.query_workload.language { + QueryLanguage::PromQL => lower_promql_roots(workload), + QueryLanguage::SQL(_) | QueryLanguage::DataFusion => lower_sql_roots(workload, catalog), + other => Err(format!("unsupported query language {other:?}")), + } +} + +/// `capabilities` as `asap_planner` plans with them: the executor's summary +/// families when the deployment declares none. +pub fn with_executor_summaries(mut capabilities: DeploymentCapabilities) -> DeploymentCapabilities { + if capabilities.supported_summaries.is_none() { + capabilities.supported_summaries = Some(asap_executor::supported_summary_families()); + } + capabilities +} + +/// The models `asap_planner` plans with: the built-in accuracy evidence, the +/// deployment's cost inputs and capabilities (see +/// [`with_executor_summaries`]), and whether accuracy is enforced. +pub fn planning_models<'a>( + capabilities: &'a DeploymentCapabilities, + cost_inputs: &'a CostInputs, + ignore_accuracy_guarantees: bool, +) -> PlanningModels<'a> { + PlanningModels::builtin() + .with_cost_inputs(cost_inputs) + .with_deployment_capabilities(capabilities) + .with_ignore_accuracy_guarantees(ignore_accuracy_guarantees) +} + /// Why [`run_planning_stages`] produced no run. #[derive(Debug, Clone, PartialEq)] pub enum PlanningStagesError { diff --git a/crates/devtools/tests/corpus_snapshots.rs b/crates/devtools/tests/corpus_snapshots.rs index b63d9277a..dca819d7b 100644 --- a/crates/devtools/tests/corpus_snapshots.rs +++ b/crates/devtools/tests/corpus_snapshots.rs @@ -5,9 +5,10 @@ //! with the committed one, and any change fails until the snapshot is //! regenerated and the change explained in the PR. //! -//! Each `tests/corpus_snapshots//corpus.json` declares the shared -//! planning inputs and the queries by stable id; `.json` beside it is -//! that query's snapshot. Regenerate with +//! `tests/corpus_snapshots/o11y_bench/corpus.json` declares the shared +//! planning inputs and the queries by stable id; the corpus registry +//! (`asap_devtools::corpus_registry::o11y_bench`) reads it, one workload per +//! query, and `.json` beside it is that query's snapshot. Regenerate with //! `UPDATE_SNAPSHOTS=1 cargo test --release -p asap-devtools --test corpus_snapshots -- --ignored`. //! //! Each query is planned by `asap_planner --run-until stage3`: one exact @@ -26,6 +27,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; +use asap_devtools::corpus_registry; use serde_json::{json, Map, Value}; #[path = "sketch_plan_snapshots.rs"] @@ -40,28 +42,6 @@ fn corpora_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/corpus_snapshots") } -/// `query`'s one-query workload under `corpus`'s shared inputs. -fn workload(corpus: &Value, query: &str) -> Value { - let mut entry = corpus["query_workload"]["repeating_query"].clone(); - entry["query"] = json!(query); - json!({ - "query_workload": { - "language": corpus["query_workload"]["language"], - "repeating_queries": [entry], - }, - "data_workload": corpus["data_workload"], - }) -} - -/// Runs `asap_planner` on `workload`; panics (a planner crash) on failure. -fn plan(workload: &Value, args: &[String], scratch: &Path) -> Value { - let input = scratch.join("workload.json"); - std::fs::write(&input, workload.to_string()).unwrap(); - let mut input_args = vec!["--workload".to_owned(), input.display().to_string()]; - input_args.extend_from_slice(args); - asap_planner(&input_args, scratch) -} - /// Runs `asap_planner --run-until stage3 --format json` with `args`, which /// name its input (paths relative to the repository root). A run whose plan /// selection failed still writes its document, with Stage 3's `error`, @@ -216,67 +196,49 @@ fn corpus_plans_match_snapshots() { std::fs::create_dir_all(&scratch).unwrap(); let mut failures = Vec::new(); - let mut corpora: Vec<_> = std::fs::read_dir(corpora_dir()) - .unwrap() - .map(|entry| entry.unwrap().path()) + // The registry's o11y benchmark workloads, one per checked query. + let name = "o11y_bench"; + let dir = corpora_dir().join(name); + let _ = std::fs::remove_dir_all(regenerated.join(name)); + let workloads = corpus_registry::o11y_bench().unwrap(); + let ids: Vec<&str> = workloads + .iter() + .map(|w| w.id.strip_prefix("o11y_bench/").unwrap()) .collect(); - corpora.sort(); - for dir in corpora { - let name = dir.file_name().unwrap().to_string_lossy().into_owned(); - let _ = std::fs::remove_dir_all(regenerated.join(&name)); - let corpus: Value = - serde_json::from_str(&std::fs::read_to_string(dir.join("corpus.json")).unwrap()) - .unwrap(); - let args: Vec = - serde_json::from_value(corpus["asap_planner_args"].clone()).unwrap(); - // A query with a `skip_reason` is not planned. - let queries: Vec<&Value> = corpus["queries"] - .as_array() - .unwrap() - .iter() - .filter(|q| q.get("skip_reason").is_none()) - .collect(); - let ids: Vec<&str> = queries.iter().map(|q| q["id"].as_str().unwrap()).collect(); - // A snapshot whose query left the manifest, or became known-slow, - // would otherwise go stale unseen. - for entry in std::fs::read_dir(&dir).unwrap() { - let file = entry.unwrap().file_name().to_string_lossy().into_owned(); - if let Some(id) = file.strip_suffix(".json") { - if id != "corpus" && !ids.contains(&id) { - failures.push(format!("{name}/{file}: no checked query with id {id}")); - } + // A snapshot whose query left the manifest, or became known-slow, + // would otherwise go stale unseen. + for entry in std::fs::read_dir(&dir).unwrap() { + let file = entry.unwrap().file_name().to_string_lossy().into_owned(); + if let Some(id) = file.strip_suffix(".json") { + if id != "corpus" && !ids.contains(&id) { + failures.push(format!("{name}/{file}: no checked query with id {id}")); } } + } - for query in queries { - let id = query["id"].as_str().unwrap(); - let document = plan( - &workload(&corpus, query["query"].as_str().unwrap()), - &args, - &scratch, - ); - let actual = serde_json::to_string_pretty(&snapshot_of(&document)).unwrap() + "\n"; - let path = dir.join(format!("{id}.json")); - if update { - std::fs::write(&path, &actual).unwrap(); - continue; - } - let expected = std::fs::read_to_string(&path).unwrap_or_default(); - if expected != actual { - let out = regenerated.join(&name); - std::fs::create_dir_all(&out).unwrap(); - std::fs::write(out.join(format!("{id}.json")), &actual).unwrap(); - std::fs::write( - out.join(format!("{id}.document.json")), - serde_json::to_string_pretty(&document).unwrap(), - ) - .unwrap(); - failures.push(format!( - "{name}/{id}: plan changed at {}", - first_difference(&expected, &actual) - )); - } + for (workload, id) in workloads.iter().zip(&ids) { + let document = asap_planner(&workload.asap_planner_args(&scratch).unwrap(), &scratch); + let actual = serde_json::to_string_pretty(&snapshot_of(&document)).unwrap() + "\n"; + let path = dir.join(format!("{id}.json")); + if update { + std::fs::write(&path, &actual).unwrap(); + continue; + } + let expected = std::fs::read_to_string(&path).unwrap_or_default(); + if expected != actual { + let out = regenerated.join(name); + std::fs::create_dir_all(&out).unwrap(); + std::fs::write(out.join(format!("{id}.json")), &actual).unwrap(); + std::fs::write( + out.join(format!("{id}.document.json")), + serde_json::to_string_pretty(&document).unwrap(), + ) + .unwrap(); + failures.push(format!( + "{name}/{id}: plan changed at {}", + first_difference(&expected, &actual) + )); } } std::fs::remove_dir_all(&scratch).unwrap(); @@ -291,6 +253,18 @@ fn corpus_plans_match_snapshots() { ); } +/// `corpus_plans_match_snapshots` plans and checks only `o11y_bench`, so a +/// second corpus directory would never be planned nor its snapshots +/// checked: add it to the corpus registry and the test first. +#[test] +fn the_corpus_directory_holds_only_the_o11y_benchmark() { + let entries: Vec = std::fs::read_dir(corpora_dir()) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(entries, ["o11y_bench"]); +} + /// The snapshot keeps the plan without the selection MILP's size and /// timings, and rounds floats. #[test] diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/corpus.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/corpus.json index 00255dfd0..e23786db5 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/corpus.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/corpus.json @@ -1,6 +1,5 @@ { "source": "crates/frontend-promql/tests/observability/data/o11y_bench_promql.txt", - "asap_planner_args": [], "query_workload": { "language": "promql", "repeating_query": { diff --git a/crates/devtools/tests/planner_fuzz.rs b/crates/devtools/tests/planner_fuzz.rs index dd1ac373e..4cd8ef476 100644 --- a/crates/devtools/tests/planner_fuzz.rs +++ b/crates/devtools/tests/planner_fuzz.rs @@ -7,19 +7,21 @@ use std::path::Path; +use asap_devtools::corpus_registry::fuzz_seed_workloads; use asap_devtools::planner_fuzz::generate::{ OuterAggregation, OuterOp, PromqlSpec, QuerySpec, RangeFunction, SqlAggregate, SqlSpec, }; use asap_devtools::planner_fuzz::relations::{evaluate, Check, Outcome, Relation, Witness}; -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; +use asap_devtools::planner_fuzz::{FuzzSettings, FuzzWorkload, Planner}; +/// The corpus registry's seed `fuzz_seeds/`. fn seed(name: &str) -> FuzzWorkload { - read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/planner_fuzz_seeds") - .join(name), - ) - .expect("the seed file reads") + let id = format!("fuzz_seeds/{name}"); + fuzz_seed_workloads() + .expect("the seeds read") + .into_iter() + .find_map(|(seed_id, workload)| (seed_id == id).then_some(workload)) + .unwrap_or_else(|| panic!("no registry seed {id}")) } fn check(relation: Relation, workload: FuzzWorkload, target: usize) -> Outcome { @@ -41,7 +43,7 @@ fn check(relation: Relation, workload: FuzzWorkload, target: usize) -> Outcome { /// finer-aggregate reuse (#950), and is now its regression test. #[test] fn finds_issue_902_filter_on_group_key_not_reading_the_grouped_result() { - let mut workload = seed("issue902.json"); + let mut workload = seed("issue902"); let per_city = SqlSpec { table: "metrics".into(), aggregate: SqlAggregate::Quantile(0.9), @@ -64,7 +66,7 @@ fn finds_issue_902_filter_on_group_key_not_reading_the_grouped_result() { /// the grouped rate result. Open bug: flip to `Holds` when #891 is fixed. #[test] fn finds_issue_891_filtered_rate_not_reading_the_grouped_rate() { - let mut workload = seed("issue891.workload.json"); + let mut workload = seed("issue891"); let grouped = PromqlSpec { metric: "container_cpu_usage_seconds_total".into(), matchers: vec![], @@ -99,7 +101,7 @@ fn finds_issue_891_filtered_rate_not_reading_the_grouped_rate() { /// A relation that holds: an exact duplicate of a query the planner shares. #[test] fn exact_duplicate_of_a_shared_query_holds() { - let mut workload = seed("issue903.workload.json"); + let mut workload = seed("issue903"); workload.queries.truncate(1); assert!( matches!( @@ -123,7 +125,7 @@ fn campaign_writes_summary_coverage_and_drafts() { cases: 2, time_budget_s: 600.0, relations: vec![Relation::ExactDuplicate, Relation::OtherRepeatInterval], - seeds: vec![seed("issue903.workload.json")], + seeds: vec![seed("issue903")], out_dir: out.clone(), max_shrink_attempts: 5, // The report test must honor case_timeout_s even when an individual @@ -408,7 +410,7 @@ fn other_frequency_statistic_witness_costs_from_one_univmon() { /// selected it for: the witness's reference point is on the same footing. #[test] fn the_selected_plan_costs_what_the_search_selected_it_for() { - let mut workload = seed("issue903.workload.json"); + let mut workload = seed("issue903"); workload.queries.truncate(1); let mut planner = Planner::new(FuzzSettings::default()); let planned = planner.plan(&workload).expect("W plans"); diff --git a/crates/devtools/tests/sketch_plan_snapshots.rs b/crates/devtools/tests/sketch_plan_snapshots.rs index 5d67f43f3..69eaf04ac 100644 --- a/crates/devtools/tests/sketch_plan_snapshots.rs +++ b/crates/devtools/tests/sketch_plan_snapshots.rs @@ -9,7 +9,8 @@ //! sizes, sharing, timing, lifetime or cost shows up as a snapshot //! diff. //! -//! `tests/sketch_plan_snapshots/manifest.json` lists each workload by id: +//! `tests/sketch_plan_snapshots/manifest.json` lists each workload by id +//! (read by `asap_devtools::corpus_registry::sketch_plans`): //! its `workload` file relative to the repository root, SQL `tables`, //! `deployment` (the `asap_planner` arguments of its deployment //! capabilities and cost inputs, files in `deployments/`), and whether @@ -26,44 +27,18 @@ use std::path::PathBuf; use serde_json::{json, Value}; -use super::{asap_planner, first_difference, first_differing_path, plan_snapshot, rounded}; +use asap_cost::CostInputs; +use asap_devtools::corpus_registry; +use asap_types::deployment_capabilities::DeploymentCapabilities; -fn repository_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") -} +use super::{asap_planner, first_difference, first_differing_path, plan_snapshot, rounded}; fn snapshots_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/sketch_plan_snapshots") } -/// The `asap_planner` arguments for `entry` under `manifest`'s deployments. -fn arguments(manifest: &Value, entry: &Value) -> Vec { - let path = |key: &str| repository_root().join(entry[key].as_str().unwrap()); - let mut args = Vec::new(); - args.extend([ - "--workload".to_owned(), - path("workload").display().to_string(), - ]); - if entry.get("tables").is_some() { - let tables: Vec = - serde_json::from_str(&std::fs::read_to_string(path("tables")).unwrap()).unwrap(); - for table in tables { - args.extend(["--sql-catalog-table".to_owned(), table.to_string()]); - } - } - let deployment = entry["deployment"].as_str().unwrap(); - let deployment: Vec = - serde_json::from_value(manifest["deployments"][deployment].clone()) - .unwrap_or_else(|_| panic!("{entry}: unknown deployment {deployment}")); - args.extend(deployment); - match entry["accuracy_guarantees"].as_str() { - None | Some("enforced") => {} - Some("ignored") => args.push("--ignore-accuracy-guarantees".to_owned()), - Some(other) => { - panic!("{entry}: accuracy_guarantees must be enforced or ignored, not {other}") - } - } - args +fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") } /// The stable part of an `asap_planner` document: every query's text and @@ -97,10 +72,12 @@ fn sketch_plans_match_snapshots() { std::env::temp_dir().join(format!("sketch_plan_snapshots_{}", std::process::id())); std::fs::create_dir_all(&scratch).unwrap(); let dir = snapshots_dir(); - let manifest: Value = - serde_json::from_str(&std::fs::read_to_string(dir.join("manifest.json")).unwrap()).unwrap(); - let entries = manifest["workloads"].as_array().unwrap(); - let ids: Vec<&str> = entries.iter().map(|e| e["id"].as_str().unwrap()).collect(); + // The registry's sketch plan workloads, read from `manifest.json`. + let workloads = corpus_registry::sketch_plans().unwrap(); + let ids: Vec<&str> = workloads + .iter() + .map(|w| w.id.strip_prefix("sketch_plans/").unwrap()) + .collect(); let mut failures = Vec::new(); // A snapshot whose workload left the manifest would otherwise go stale. @@ -113,9 +90,8 @@ fn sketch_plans_match_snapshots() { } } - for entry in entries { - let id = entry["id"].as_str().unwrap(); - let document = asap_planner(&arguments(&manifest, entry), &scratch); + for (workload, id) in workloads.iter().zip(&ids) { + let document = asap_planner(&workload.asap_planner_args(&scratch).unwrap(), &scratch); let actual_text = serde_json::to_string_pretty(&snapshot_of(&document)).unwrap() + "\n"; let path = dir.join(format!("{id}.json")); if update { @@ -158,28 +134,52 @@ fn sketch_plans_match_snapshots() { ); } -/// Each manifest entry names a known deployment and an existing input, and -/// builds its arguments. +/// Each manifest entry loads into the registry with the capabilities and +/// cost inputs of the files its deployment names, and its accuracy setting. #[test] -fn manifest_entries_build_their_arguments() { - let manifest: Value = serde_json::from_str( - &std::fs::read_to_string(snapshots_dir().join("manifest.json")).unwrap(), - ) - .unwrap(); - for entry in manifest["workloads"].as_array().unwrap() { - for key in ["workload", "tables"] { - if let Some(path) = entry[key].as_str() { - assert!(repository_root().join(path).is_file(), "{entry}: no {path}"); - } - } - let args = arguments(&manifest, entry); - for path in args.iter().skip(2).filter(|arg| arg.ends_with(".json")) { - assert!(repository_root().join(path).is_file(), "{entry}: no {path}"); - } +fn manifest_entries_load_with_their_deployments() { + let read = |path: &str| -> Value { + serde_json::from_str(&std::fs::read_to_string(repository_root().join(path)).unwrap()) + .unwrap() + }; + let manifest = read("crates/devtools/tests/sketch_plan_snapshots/manifest.json"); + let entries = manifest["workloads"].as_array().unwrap(); + let workloads = corpus_registry::sketch_plans().unwrap(); + assert_eq!(workloads.len(), entries.len()); + for (entry, workload) in entries.iter().zip(&workloads) { assert_eq!( - args.iter().any(|a| a == "--ignore-accuracy-guarantees"), + workload.id, + format!("sketch_plans/{}", entry["id"].as_str().unwrap()) + ); + assert_eq!( + workload.ignore_accuracy_guarantees, entry["accuracy_guarantees"] == json!("ignored") ); + let deployment = manifest["deployments"][entry["deployment"].as_str().unwrap()] + .as_array() + .unwrap(); + let file_after = |flag: &str| -> &str { + let at = deployment.iter().position(|arg| arg == flag).unwrap(); + deployment[at + 1].as_str().unwrap() + }; + assert_eq!( + serde_json::to_value(&workload.deployment_capabilities).unwrap(), + serde_json::to_value( + serde_json::from_value::(read(file_after( + "--deployment-capabilities" + ))) + .unwrap() + ) + .unwrap(), + "{}", + workload.id + ); + assert_eq!( + workload.cost_inputs, + serde_json::from_value::(read(file_after("--cost-inputs"))).unwrap(), + "{}", + workload.id + ); } } diff --git a/crates/integration-tests/tests/milp_objective.rs b/crates/integration-tests/tests/milp_objective.rs index 4d2e03a65..21122815a 100644 --- a/crates/integration-tests/tests/milp_objective.rs +++ b/crates/integration-tests/tests/milp_objective.rs @@ -1,21 +1,14 @@ //! The selection MILP models the selected plan's cost exactly: its optimal //! objective is `asap_cost::plan_cost` of the plan it builds, under cost //! model 1 (concepts doc, section 5, "Cost and retention"). -use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; -use asap_planner::PlanningModels; -use asap_types::workload::PlanningWorkload; +use asap_cost::CostInputs; +use asap_devtools::corpus_registry::{plan_workload, repeating_promql_workload}; +use asap_devtools::planning_stages::RunUntil; +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::workload::{DurationMs, PlanningWorkload, PrometheusMetricType, SourceFacts}; use serde_json::json; -/// A query `query` every 10 s at phase 0, within 1 % at δ = 0.01. -fn every_10s(query: &str) -> serde_json::Value { - json!({ - "query": query, - "demand": {"fixed_interval_at": {"interval": 10000, "evaluation_phase": 0}}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.01}}, - "predictability": {"predictable": {"known_at": null}}, - }) -} - /// Two windowed quantiles over one metric (1 m and 5 m, every 10 s; 10,000 /// series every 15 s), accuracy not enforced: the selected plan shares /// one query-time scan, whose execution reads the 5 m window once and @@ -23,25 +16,24 @@ fn every_10s(query: &str) -> serde_json::Value { /// cost to 1e-6 of it. #[test] fn the_milp_objective_is_the_plan_cost_on_a_shared_scan() { - let workload: PlanningWorkload = serde_json::from_value(json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [ - every_10s("quantile_over_time(0.5, data[1m])"), - every_10s("quantile_over_time(0.9, data[5m])"), - ], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}, - }, - })) - .unwrap(); - let plan_cost = milp_objective_equals_plan_cost( - &workload, - PlanningModels::builtin().with_ignore_accuracy_guarantees(true), + let workload = repeating_promql_workload( + &[ + "quantile_over_time(0.5, data[1m])", + "quantile_over_time(0.9, data[5m])", + ], + 10_000, + AccuracyTarget::max_error(0.01).failure_probability(0.01), + &[( + "data", + SourceFacts { + series_count: Some(10_000), + metric_type: Some(PrometheusMetricType::Gauge), + sample_interval: Some(DurationMs(15_000)), + ..SourceFacts::default() + }, + )], ); + let plan_cost = milp_objective_equals_plan_cost(&workload, true); assert!(plan_cost <= 0.144, "{plan_cost}"); } @@ -72,22 +64,34 @@ fn the_milp_objective_is_the_plan_cost_on_an_offset_range_query() { }, })) .unwrap(); - milp_objective_equals_plan_cost(&workload, PlanningModels::builtin()); + milp_objective_equals_plan_cost(&workload, false); } -/// Plans `workload` and checks the MILP's objective equals `plan_cost` of -/// the selected plan to 1e-6 of it; returns that cost per second. -fn milp_objective_equals_plan_cost(workload: &PlanningWorkload, models: PlanningModels) -> f64 { - let roots = lower_promql_roots(workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap(); - let run = run_planning_stages_over(roots, &entries, &data, models).unwrap(); - let (plan, selection) = run.run.selection.as_ref().expect("a plan is selected"); +/// Plans `workload` (accuracy not enforced when `ignore_accuracy`) on the +/// default deployment and checks the MILP's objective equals `plan_cost` +/// of the selected plan to 1e-6 of it; returns that cost per second. +fn milp_objective_equals_plan_cost(workload: &PlanningWorkload, ignore_accuracy: bool) -> f64 { + let cost_inputs = CostInputs::default(); + let planned = plan_workload( + workload, + &[], + &DeploymentCapabilities::default(), + &cost_inputs, + ignore_accuracy, + RunUntil::Stage3, + ) + .unwrap(); + let (plan, selection) = planned + .stages + .run + .selection + .as_ref() + .expect("a plan is selected"); let plan_cost = asap_integration_tests::planning::plan_cost_per_second( plan, - &entries, - &data, - &asap_cost::CostInputs::default(), + &planned.entries, + &planned.data, + &cost_inputs, ); assert!( (selection.milp_objective - plan_cost).abs() <= 1e-6 * plan_cost, diff --git a/docs/develop_docs/planner-snapshots.md b/docs/develop_docs/planner-snapshots.md index 04986939e..ebaa1e0c4 100644 --- a/docs/develop_docs/planner-snapshots.md +++ b/docs/develop_docs/planner-snapshots.md @@ -11,8 +11,9 @@ regenerated and the change is explained. `crates/devtools/tests/corpus_snapshots//`: -- `corpus.json`: the inputs shared by the corpus's queries (`asap_planner_args`, - the repeating-query entry and the `data_workload`), and the queries, each +- `corpus.json`: the inputs shared by the corpus's queries (the + repeating-query entry and the `data_workload`; every query is planned on + the default deployment), and the queries, each with a stable `id`. Each query is planned on its own. A query with a `skip_reason` entry (saying why) is skipped and has no snapshot. - `.json`: that query's snapshot, pretty-printed with sorted keys. diff --git a/docs/develop_docs/testing.md b/docs/develop_docs/testing.md index b889a0e8b..462d584ef 100644 --- a/docs/develop_docs/testing.md +++ b/docs/develop_docs/testing.md @@ -33,7 +33,12 @@ chain rule over `quantile_over_time(0.9, m[5m])` every 60 s adds one `SummaryTumblingWindowChain` group with width 60 s; the test checks that group and its candidates, not the plan. -Build inputs only through the production frontends and Stage 0. A test that +Build inputs only through the production frontends and Stage 0: +`corpus_registry::repeating_promql_workload` writes a small workload and +`corpus_registry::plan_workload(.., until)` plans it up to the stage under +test with `asap_planner`'s own lowering and planning models +(`planning_stages::lower_workload`, `planning_models`), and Stage 3 through +the facade's `run_planning`. A test that assembles IR by hand can drop what the frontend adds (for example the series identity column) and then pass on a memo no user can produce. @@ -130,6 +135,12 @@ behavior, not the issue number. One list of in-repo workloads, each with the deployment it is planned for (capabilities, cost inputs, whether accuracy is enforced), read by the contracts, the snapshots and the oracles: +`asap_devtools::corpus_registry::corpus_workloads()`, ids +`/` (e.g. `o11y_bench/q21`). `CorpusWorkload::plan(until)` +plans one as `asap_planner --run-until ` does, and +`CorpusWorkload::asap_planner_args` writes its inputs for the CLI. +The ids' source prefixes are `corpus_registry::SOURCES`; `planner_fuzz`'s +tests read their seeds from the registry (`fuzz_seed_workloads`). | Source | Workloads | |---|---|