From 6fc40ad6608c36418f07069e0b218c3cec1dd8d3 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:14:14 +0000 Subject: [PATCH 1/6] test: plan file without the Planner executor's compile check (failing first) The selected plan written as an InstallablePlan plan file (--plan-file), with and without the executor's compile check (--executor-check), and InstallablePlan::of_dag_without_executor_check for a runtime other than the Planner's executor. --- crates/devtools/tests/asap_planner.rs | 200 ++++++++++++++++++++++++ crates/planner/src/installable/tests.rs | 48 ++++++ 2 files changed, 248 insertions(+) diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index db141c0b6..0a29f328b 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -2377,3 +2377,203 @@ fn the_default_text_is_the_terse_form_and_verbose_adds_the_details() { } } } + +// ── Plan file (`--plan-file`, `--executor-check`) ─────────────────────────── + +/// A plan-file path holding a plan file an earlier run left there: a run +/// overwrites it with its plan, or removes it when it writes none. +fn stale_plan_file() -> TempFile { + TempFile::new("a plan file an earlier run left") +} + +/// The plan file at `path`, read back as a runtime reads it and validated. +fn read_plan_file(path: &str) -> asap_planner::InstallablePlan { + let text = std::fs::read_to_string(path).unwrap(); + let plan = asap_planner::InstallablePlan::from_json(&text).unwrap(); + plan.validate().unwrap(); + assert_eq!( + plan.wire_version, + asap_planner::INSTALLABLE_PLAN_WIRE_VERSION + ); + plan +} + +/// Whether `plan`'s DAG is the plan `document` selected; the document's +/// numbers are rounded to 12 significant digits, so the file's are too. +fn is_the_selected_plan(plan: &asap_planner::InstallablePlan, document: &Value) -> bool { + asap_devtools::stable_output::rounded_to_12_significant_digits( + serde_json::to_value(&plan.dag).unwrap(), + ) == document["stage3_plan_selection"]["plan"] +} + +/// `--plan-file` writes the selected plan of #509's example 1 as +/// `InstallablePlan` JSON, with the executor's compile check (the default) +/// and without it (`skip`): the file reads back with +/// `InstallablePlan::from_json`, validates, holds the workload's query +/// texts in order, and its DAG is the plan the document selected. +#[test] +fn plan_file_round_trips_with_and_without_the_executor_check() { + let workload: PlanningWorkload = + serde_json::from_str(&std::fs::read_to_string(example("example1")).unwrap()).unwrap(); + let query_texts: Vec = workload + .query_workload + .entries() + .map(|entry| entry.query.0.clone()) + .collect(); + for check in [&[][..], &["--executor-check", "skip"]] { + let file = stale_plan_file(); + let mut args = vec!["--run-until", "stage3", "--plan-file", file.path()]; + args.extend_from_slice(check); + let document = example_document("example1", &args); + let plan = read_plan_file(file.path()); + assert_eq!(plan.query_texts, query_texts, "{check:?}"); + assert!(is_the_selected_plan(&plan, &document), "{check:?}"); + } +} + +/// `sum by (job) (mad_over_time(data[1m]))`: Stages 1-3 plan it, but the +/// Planner's executor has no `mad_over_time` kernel. Without `--plan-file` +/// the run succeeds and the document says why the plan is not installable, +/// as before. With the compile check (the default, or `compile`) and +/// `--plan-file`, the stage output is still written, the run exits 1, and +/// no plan file is left at the path; with `skip` the plan file is written +/// for a runtime other than the Planner's executor, and that plan really +/// does not compile there. +#[test] +fn executor_check_skip_writes_a_plan_the_executor_cannot_compile() { + let query = "sum by (job) (mad_over_time(data[1m]))"; + let workload = TempFile::json(&adhoc(&[query], exact(), &["data"])); + let stage3 = ["--workload", workload.path(), "--run-until", "stage3"]; + + // Precondition: an executor-only gap. Once the executor runs + // `mad_over_time`, pick another query it plans but cannot compile. + let document = document_of(&stage3); + let views_error = document["stage3_plan_selection"]["materialized_views_error"] + .as_str() + .unwrap_or_default(); + assert!(views_error.contains("the executor cannot"), "{document}"); + + for check in [&[][..], &["--executor-check", "compile"]] { + let file = stale_plan_file(); + let mut args = stage3.to_vec(); + args.extend(["--plan-file", file.path()]); + args.extend_from_slice(check); + let output = planner(&args); + assert_eq!(output.status.code(), Some(1), "{check:?}"); + assert!(!output.stdout.is_empty(), "the stage output is written"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("the executor cannot"), "{check:?}: {stderr}"); + assert!(!stderr.contains("usage:"), "{stderr}"); + assert!(!Path::new(file.path()).exists(), "{check:?}"); + } + + let file = stale_plan_file(); + let mut args = stage3.to_vec(); + args.extend(["--plan-file", file.path(), "--executor-check", "skip"]); + let document = document_of(&args); + let plan = read_plan_file(file.path()); + assert_eq!(plan.query_texts, [query]); + assert!(is_the_selected_plan(&plan, &document)); + assert!(plan.check_compiles().is_err()); + // The document derives the same plan's views, without the compile check. + let selection = &document["stage3_plan_selection"]; + assert!(selection["materialized_views_error"].is_null(), "{selection}"); + assert!(selection["materialized_views"].is_array(), "{selection}"); +} + +/// With `skip`, supported summaries the deployment leaves unrestricted +/// stay unrestricted: Pass 1 offers a family the Planner's executor does +/// not build (Count Sketch for `count`, which +/// `undeclared_supported_summaries_default_to_the_executors_families` shows +/// is not offered by default), and the document shows no restriction. +/// `compile` is the default. +#[test] +fn executor_check_skip_keeps_unrestricted_summaries_unrestricted() { + let workload = adhoc(&["count by (job) (m)"], within(0.01, 0.01), &["m"]); + let planned_with = |check: &str| { + document( + &workload, + &[ + "--run-until", + "stage1_pass1", + "--ignore-accuracy-guarantees", + "--executor-check", + check, + ], + ) + }; + let compile = planned_with("compile"); + assert_eq!( + compile["deployment_capabilities"]["supported_summaries"], + serde_json::to_value(asap_executor::supported_summary_families()).unwrap() + ); + let skip = planned_with("skip"); + assert!(skip["deployment_capabilities"]["supported_summaries"].is_null()); + let labels = memo_labels(&skip, "pass1"); + assert!( + labels.iter().any(|label| label.contains("CountSketch")), + "{labels:?}" + ); +} + +/// When Stage 3 selects no plan (example 3b under cost model 2, which plan +/// selection does not implement), `--plan-file` writes none and removes +/// the one an earlier run left, and the run exits 1. +#[test] +fn plan_file_is_not_left_when_no_plan_is_selected() { + let cost = cost_inputs(&asap_cost::CostInputs { + cost_model: asap_cost::CostModel::ProvisionedCapacity( + asap_cost::ResourceUnitCost::illustrative(), + ), + ..Default::default() + }); + let file = stale_plan_file(); + let document = example_document( + "example3b", + &[ + "--run-until", + "stage3", + "--cost-inputs", + cost.path(), + "--plan-file", + file.path(), + ], + ); + assert!(!selects_a_plan(&document["stage3_plan_selection"])); + assert!(!Path::new(file.path()).exists()); +} + +/// An unknown `--executor-check` value, `--plan-file` without its path, and +/// `--plan-file` before Stage 3 selects a plan are usage errors (status 2, +/// with the usage line) that leave the plan-file path alone. +#[test] +fn plan_file_flag_misuse_is_a_usage_error() { + let workload = TempFile::json(&adhoc(&["sum(data)"], exact(), &["data"])); + let file = stale_plan_file(); + for (args, message) in [ + ( + vec!["--run-until", "stage3", "--executor-check", "none"], + "unknown executor check none", + ), + ( + vec!["--run-until", "stage3", "--plan-file"], + "--plan-file needs a value", + ), + ( + vec!["--run-until", "stage2", "--plan-file", file.path()], + "--plan-file needs --run-until stage3", + ), + ] { + let mut all = vec!["--workload", workload.path()]; + all.extend(args); + let output = planner(&all); + assert_eq!(output.status.code(), Some(2), "{all:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains(message), "{stderr}"); + assert!(stderr.contains("usage: asap_planner"), "{stderr}"); + } + assert_eq!( + std::fs::read_to_string(file.path()).unwrap(), + "a plan file an earlier run left" + ); +} diff --git a/crates/planner/src/installable/tests.rs b/crates/planner/src/installable/tests.rs index 1386370a5..cd72fd516 100644 --- a/crates/planner/src/installable/tests.rs +++ b/crates/planner/src/installable/tests.rs @@ -372,3 +372,51 @@ fn a_root_time_shift_is_read_whole_by_the_installer_not_by_the_cost() { assert_eq!(retention.len(), 1); assert_eq!(retention[0].lookback_ms(), None); } + +/// `sum by (job) (last_over_time(data[1m]))` every 10 s, exact: Stages 1-3 +/// plan it, but the Planner's executor has no `last_over_time` kernel, so +/// its compile check refuses the plan. A runtime other than the Planner's +/// executor still gets the plan, validated, from +/// [`InstallablePlan::of_dag_without_executor_check`]. +#[tokio::test] +async fn a_plan_only_the_executor_cannot_compile_is_built_without_its_check() { + let workload: crate::PlanningWorkload = serde_json::from_str( + r#"{"query_workload": {"language": "promql", "repeating_queries": [{ + "query": "sum by (job) (last_over_time(data[1m]))", + "demand": {"fixed_interval_at": {"interval": 10000, "evaluation_phase": 0}}, + "requirements": {"accuracy": {"max_error": 0.0, "failure_probability": 0.0}}, + "predictability": {"predictable": {"known_at": null}}}]}, + "data_workload": {"arrival": {"ingesting": {"rows_per_second": 10000.0}}, + "sources": {"data": {"series_count": 1000, "metric_type": "gauge", + "sample_interval": 100}}}}"#, + ) + .unwrap(); + let output = crate::e2e_plan(crate::UserInput::new( + &workload, + crate::FrontendInput::Promql, + crate::PlanningModels::builtin(), + )) + .await + .expect("Stages 1-3 plan last_over_time"); + let query_texts = vec!["sum by (job) (last_over_time(data[1m]))".to_owned()]; + // Precondition: an executor-only gap. Once the executor runs + // `last_over_time`, pick another query it plans but cannot compile. + let refused = InstallablePlan::of_dag(output.plan.clone(), query_texts.clone()).unwrap_err(); + assert!( + matches!( + refused, + InstallablePlanError::QueryRunNotExecutable { .. } + | InstallablePlanError::MaterializedViewNotExecutable { .. } + ), + "{refused}" + ); + let plan = InstallablePlan::of_dag_without_executor_check(output.plan.clone(), query_texts) + .expect("validated without the executor's compile check"); + plan.validate().unwrap(); + assert!(plan.check_compiles().is_err()); + // Still validated: one query text per root. + assert!(matches!( + InstallablePlan::of_dag_without_executor_check(output.plan, Vec::new()), + Err(InstallablePlanError::QueriesDoNotMatchRoots { roots: 1 }) + )); +} From 4a3efdb0f78c950f9b59aa7cda0cb9cf0d8d14fb Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:20:53 +0000 Subject: [PATCH 2/6] feat: plan file without the Planner executor's compile check InstallablePlan::of_dag_without_executor_check builds the validated plan for a runtime other than the Planner's executor; of_dag is it plus the executor's compile check. asap_planner --plan-file writes the selected plan as InstallablePlan JSON, and --executor-check compile|skip says whether it must compile on the Planner's executor (default compile); with skip, unrestricted supported summaries stay unrestricted. A plan file left at the path is removed once the arguments are read. --- crates/devtools/Cargo.toml | 4 +- crates/devtools/src/bin/asap_planner.rs | 116 ++++++++++++++++++++---- crates/devtools/tests/asap_planner.rs | 69 ++++++++------ crates/planner/src/installable/mod.rs | 31 +++++-- 4 files changed, 162 insertions(+), 58 deletions(-) diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index e1559b4f1..37c4ff86d 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -14,8 +14,8 @@ asap-stage1-pass2-reuse-candidates = { path = "../stage1-pass2-reuse-candidates" asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-cost = { path = "../cost" } asap-stage3-plan-selection = { path = "../stage3-plan-selection" } -# The installable plan of the selected candidate (`installable_plan_of`), -# whose stored outputs `asap_planner` writes. +# The installable plan of the selected candidate (`InstallablePlan`), +# whose stored outputs and plan file `asap_planner` writes. asap-planner = { path = "../planner" } # The executor's supported summary families, which `asap_planner` plans # with when the deployment declares none, as `plan_installable` does. diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index d5da82ff8..97c9d6efb 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -45,12 +45,28 @@ // `state_not_mergeable_across_windows`, `excluded_candidates`; an // omitted field takes its default). // Without `supported_summaries`, the executor's summary families are -// supported, as `plan_installable` defaults it. +// supported, as `plan_installable` defaults it, unless +// `--executor-check skip`. // - `--ignore-accuracy-guarantees`: Stage 3 accepts every query output // regardless of its accuracy guarantee // (`PlanningModels::ignore_accuracy_guarantees`), comparing candidates // on cost alone. By default it enforces them, as `plan_installable` // does. +// - `--executor-check compile|skip`: whether the selected plan must +// compile on the Planner's executor (`compile`, the default, as +// `plan_installable` checks it), or is for a runtime other than the +// Planner's executor (`skip`): then unrestricted `supported_summaries` +// stay unrestricted, so the deployment capabilities passed are the only +// restriction, and the plan is validated +// (`InstallablePlan::of_dag_without_executor_check`) but not compiled. +// +// Besides the stage output, `--plan-file ` (with `--run-until +// stage3`) writes the selected plan as an `InstallablePlan` JSON plan file, +// which a runtime reads with `InstallablePlan::from_json`, checked as +// `--executor-check` says. When no plan is selected or the plan fails the +// check, no plan file is written and the run exits 1 after writing the +// stage output. A plan file left at the path is removed once the arguments +// are read, so no failed run leaves one there. // // The document (`--format json`): // - workload: the queries (`id`, `language`, `text`, `requirements`, @@ -82,7 +98,8 @@ // `parameters`, `timing`, `lifetime`) and the `queries` reading it. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them; `materialized_views_error` when the -// plan is not installable. `warnings`: a PromQL query applying `rate`, `increase` +// plan is not installable (under `--executor-check skip`, not checked +// against the Planner's executor). `warnings`: a PromQL query applying `rate`, `increase` // or `irate` to a declared gauge or combining a declared summary's // quantile series across series (`metric_type_warnings`), each data fact the selected // plan assumed because the data workload does not declare it (`assumed_data_facts`, @@ -137,7 +154,7 @@ use asap_devtools::planning_stages_text::{ use asap_devtools::stable_output::{rounded_to_12_significant_digits, CanonicalIds}; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_planner::{assumed_data_facts, installable_plan_of, PlanningModels, SelectionError}; +use asap_planner::{assumed_data_facts, InstallablePlan, PlanningModels, SelectionError}; use asap_stage3_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; use asap_types::accuracy::{default_metric_for_root, AccuracyTarget}; use asap_types::deployment_capabilities::DeploymentCapabilities; @@ -160,7 +177,7 @@ use serde_json::{json, Value}; const USAGE: &str = "usage: asap_planner (--workload )... \ --run-until stage0|stage1_pass1|stage1_pass2|stage2|stage3 [--format text|json] [--verbose] [--out ] \ [--sql-catalog-table ]... [--cost-inputs ] [--deployment-capabilities ] \ -[--ignore-accuracy-guarantees]"; +[--ignore-accuracy-guarantees] [--executor-check compile|skip] [--plan-file ]"; fn main() { match run(std::env::args().skip(1).collect()) { @@ -189,6 +206,17 @@ impl From for Failure { } } +/// Whether the selected plan must compile on the Planner's executor +/// (`--executor-check`). +#[derive(Clone, Copy, PartialEq, Eq)] +enum ExecutorCheck { + /// It must, as `plan_installable` checks it (the default). + Compile, + /// It is for a runtime other than the Planner's executor: validated, + /// not compiled, and planned with the summaries the deployment declares. + Skip, +} + /// `path`'s contents as `T`, errors naming `flag` and `path`. fn read_json(flag: &str, path: &str) -> Result { let text = std::fs::read_to_string(path).map_err(|e| format!("{flag} {path}: {e}"))?; @@ -205,6 +233,8 @@ fn run(args: Vec) -> Result<(), Failure> { let mut deployment_capabilities = DeploymentCapabilities::default(); let mut ignore_accuracy_guarantees = false; let mut verbose = false; + let mut executor_check = ExecutorCheck::Compile; + let mut plan_file = None; let mut args = args.into_iter(); while let Some(flag) = args.next() { let mut value = || { @@ -234,13 +264,37 @@ fn run(args: Vec) -> Result<(), Failure> { "--printing-full-candidates was renamed to --verbose".into(), )) } + "--executor-check" => { + executor_check = match value()?.as_str() { + "compile" => ExecutorCheck::Compile, + "skip" => ExecutorCheck::Skip, + other => return Err(Failure::Usage(format!("unknown executor check {other}"))), + } + } + "--plan-file" => plan_file = Some(value()?), other => return Err(Failure::Usage(format!("unknown argument {other}"))), } } let run_until = run_until.ok_or(Failure::Usage("--run-until is required".into()))?; + if plan_file.is_some() && run_until != RunUntil::Stage3 { + return Err(Failure::Usage( + "--plan-file needs --run-until stage3".into(), + )); + } if workload_paths.is_empty() { return Err(Failure::Usage("--workload is required".into())); } + // A plan file an earlier run left at the path is removed first, so a + // runtime never installs a plan this run did not select, whichever way + // the run fails. + if let Some(path) = &plan_file { + match std::fs::remove_file(path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("--plan-file {path}: {e}").into()) + } + _ => {} + } + } let mut workloads = Vec::new(); for path in &workload_paths { let workload: PlanningWorkload = read_json("--workload", path)?; @@ -251,8 +305,11 @@ fn run(args: Vec) -> Result<(), Failure> { } let (catalog, table_names) = sql_catalog(&tables)?; let (described, roots, entries, data) = lowered_together(&workloads, &catalog, &table_names)?; - // Plan with the executor's summary families when the deployment declares none. - if deployment_capabilities.supported_summaries.is_none() { + // Plan with the executor's summary families when the deployment declares + // none, unless the plan is for a runtime other than the Planner's executor. + if executor_check == ExecutorCheck::Compile + && deployment_capabilities.supported_summaries.is_none() + { deployment_capabilities.supported_summaries = Some(asap_executor::supported_summary_families()); } @@ -268,14 +325,19 @@ fn run(args: Vec) -> Result<(), Failure> { .and_then(|run| run.run.selection.as_ref().ok()) .and_then(|(plan, _)| asap_cost::plan_cost(plan, &entries, &data, &cost_inputs).ok()) .map(|(cost, _)| cost); - // The selected plan's materialized views, as `plan_installable` derives them. - let materialized_views = result + // The selected plan as installed, checked as `--executor-check` says. + let installable = result .as_ref() .ok() .and_then(|run| run.run.selection.as_ref().ok()) - .map(|(plan, _)| materialized_views_of(plan, &entries)); + .map(|(plan, _)| installable_plan(plan, &entries, executor_check)); + // Its materialized views, as `plan_installable` derives them. + let materialized_views = installable.as_ref().map(|installable| { + let plan = installable.as_ref().map_err(Clone::clone)?; + plan.materialized_views().map_err(|e| e.to_string()) + }); // A stage that failed: the output is still written, then the run exits 1. - let failure = match &result { + let mut failure = match &result { Err(PlanningStagesError::Pass1(error)) => Some(format!("Pass 1: {error}")), Ok(run) => match &run.run.selection { Err(SelectionError::NotRun) | Ok(_) => None, @@ -355,6 +417,17 @@ fn run(args: Vec) -> Result<(), Failure> { Some(path) => std::fs::write(&path, text).map_err(|e| format!("{path}: {e}"))?, None => print!("{text}"), } + if let Some(path) = plan_file { + match installable { + Some(Ok(plan)) => { + let json = serde_json::to_string_pretty(&plan).map_err(|e| e.to_string())?; + std::fs::write(&path, json + "\n").map_err(|e| format!("{path}: {e}"))?; + } + Some(Err(error)) => failure = Some(format!("--plan-file {path}: {error}")), + // No plan was selected: `failure` already says why. + None => {} + } + } match failure { Some(error) => Err(Failure::Other(error)), None => Ok(()), @@ -702,16 +775,23 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option Result, String> { - let plan = installable_plan_of(plan.clone(), entries).map_err(|e| e.to_string())?; - plan.materialized_views().map_err(|e| e.to_string()) + executor_check: ExecutorCheck, +) -> Result { + let query_texts = entries.iter().map(|entry| entry.query.0.clone()).collect(); + let plan = InstallablePlan::of_dag_without_executor_check(plan.clone(), query_texts) + .map_err(|e| e.to_string())?; + if executor_check == ExecutorCheck::Compile { + plan.check_compiles()?; + } + Ok(plan) } /// Stage 2's physical candidates $\mathcal C$: each with its logical diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 0a29f328b..e11c82343 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -2406,7 +2406,8 @@ fn is_the_selected_plan(plan: &asap_planner::InstallablePlan, document: &Value) ) == document["stage3_plan_selection"]["plan"] } -/// `--plan-file` writes the selected plan of #509's example 1 as +/// `--plan-file` writes the selected plan of the committed `example1` +/// workload as /// `InstallablePlan` JSON, with the executor's compile check (the default) /// and without it (`skip`): the file reads back with /// `InstallablePlan::from_json`, validates, holds the workload's query @@ -2462,7 +2463,10 @@ fn executor_check_skip_writes_a_plan_the_executor_cannot_compile() { assert_eq!(output.status.code(), Some(1), "{check:?}"); assert!(!output.stdout.is_empty(), "the stage output is written"); let stderr = String::from_utf8(output.stderr).unwrap(); - assert!(stderr.contains("the executor cannot"), "{check:?}: {stderr}"); + assert!( + stderr.contains("the executor cannot"), + "{check:?}: {stderr}" + ); assert!(!stderr.contains("usage:"), "{stderr}"); assert!(!Path::new(file.path()).exists(), "{check:?}"); } @@ -2477,43 +2481,52 @@ fn executor_check_skip_writes_a_plan_the_executor_cannot_compile() { assert!(plan.check_compiles().is_err()); // The document derives the same plan's views, without the compile check. let selection = &document["stage3_plan_selection"]; - assert!(selection["materialized_views_error"].is_null(), "{selection}"); + assert!( + selection["materialized_views_error"].is_null(), + "{selection}" + ); assert!(selection["materialized_views"].is_array(), "{selection}"); } -/// With `skip`, supported summaries the deployment leaves unrestricted -/// stay unrestricted: Pass 1 offers a family the Planner's executor does -/// not build (Count Sketch for `count`, which -/// `undeclared_supported_summaries_default_to_the_executors_families` shows -/// is not offered by default), and the document shows no restriction. -/// `compile` is the default. +/// The deployment capabilities `asap_planner` plans with (the document +/// shows the value it hands to planning): supported summaries the +/// deployment leaves unrestricted are the executor's families under the +/// compile check (the default, as `plan_installable` defaults them) and +/// stay unrestricted with `skip`, so the capabilities the caller passes are +/// the only restriction. #[test] fn executor_check_skip_keeps_unrestricted_summaries_unrestricted() { let workload = adhoc(&["count by (job) (m)"], within(0.01, 0.01), &["m"]); - let planned_with = |check: &str| { - document( - &workload, - &[ - "--run-until", - "stage1_pass1", - "--ignore-accuracy-guarantees", - "--executor-check", - check, - ], - ) + let planned_with = |args: &[&str]| { + let mut all = vec!["--run-until", "stage0"]; + all.extend_from_slice(args); + document(&workload, &all)["deployment_capabilities"]["supported_summaries"].clone() }; - let compile = planned_with("compile"); assert_eq!( - compile["deployment_capabilities"]["supported_summaries"], + planned_with(&["--executor-check", "compile"]), serde_json::to_value(asap_executor::supported_summary_families()).unwrap() ); - let skip = planned_with("skip"); - assert!(skip["deployment_capabilities"]["supported_summaries"].is_null()); - let labels = memo_labels(&skip, "pass1"); - assert!( - labels.iter().any(|label| label.contains("CountSketch")), - "{labels:?}" + assert_eq!( + planned_with(&["--executor-check", "compile"]), + planned_with(&[]) ); + assert!(planned_with(&["--executor-check", "skip"]).is_null()); +} + +/// A run that fails before planning (here an unreadable workload file) +/// still removes the plan file an earlier run left at `--plan-file`. +#[test] +fn a_failed_run_removes_the_plan_file_left_at_the_path() { + let file = stale_plan_file(); + refused(&[ + "--workload", + "/nonexistent/workload.json", + "--run-until", + "stage3", + "--plan-file", + file.path(), + ]); + assert!(!Path::new(file.path()).exists()); } /// When Stage 3 selects no plan (example 3b under cost model 2, which plan diff --git a/crates/planner/src/installable/mod.rs b/crates/planner/src/installable/mod.rs index 166f9ecad..ec0fa6075 100644 --- a/crates/planner/src/installable/mod.rs +++ b/crates/planner/src/installable/mod.rs @@ -304,25 +304,36 @@ impl InstallablePlan { executor_compiles(&self.dag, &views).map_err(|error| error.to_string()) } - /// The plan of `dag`, with each view's and query-time scan's retention - /// (which Stage 3 set) checked ([`PhysicalASAPDAG::materialized_views`], - /// [`PhysicalASAPDAG::raw_data_retention`]), and each view's refresh - /// and the query run compiled ([`executor_compiles`]): a plan the - /// executor cannot run is refused here, not installed. + /// The plan of `dag`, validated ([`Self::of_dag_without_executor_check`]), + /// with each view's refresh and the query run compiled + /// ([`executor_compiles`]): a plan the Planner's executor cannot run is + /// refused here, not installed. pub(crate) fn of_dag( dag: PhysicalASAPDAG, query_texts: Vec, ) -> Result { - let views = dag.materialized_views()?; - dag.raw_data_retention()?; - check_queries(&dag, &query_texts)?; - executor_compiles(&dag, &views)?; + let plan = Self::of_dag_without_executor_check(dag, query_texts)?; + executor_compiles(&plan.dag, &plan.materialized_views()?)?; + Ok(plan) + } + + /// The plan of `dag` answering `query_texts` (one per root, in workload + /// order) for a runtime other than the Planner's executor: checked as + /// [`Self::validate`] checks a plan file (the DAG's validity, one text + /// per root, no standalone scalar query, and each view's and query-time + /// scan's retention, which Stage 3 set), but not compiled by the + /// Planner's executor. E.g. a plan with a `last_over_time` the executor + /// has no kernel for is returned here and refused by [`Self::of_dag`]. + pub fn of_dag_without_executor_check( + dag: PhysicalASAPDAG, + query_texts: Vec, + ) -> Result { let plan = InstallablePlan { wire_version: INSTALLABLE_PLAN_WIRE_VERSION, dag, query_texts, }; - debug_assert!(plan.validate().is_ok()); + plan.validate()?; Ok(plan) } From 40a4eb83d10995dc7acf2f2d275679366c480368 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:43:57 +0000 Subject: [PATCH 3/6] test: use absent_over_time as the executor-only example The executor now runs last_over_time, mad_over_time and ts_of_max_over_time; absent_over_time still plans in Stages 1-3 and fails the executor's compile check. --- crates/devtools/tests/asap_planner.rs | 8 ++++---- crates/planner/src/installable/mod.rs | 2 +- crates/planner/src/installable/tests.rs | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index e11c82343..3a24c6f20 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -2432,8 +2432,8 @@ fn plan_file_round_trips_with_and_without_the_executor_check() { } } -/// `sum by (job) (mad_over_time(data[1m]))`: Stages 1-3 plan it, but the -/// Planner's executor has no `mad_over_time` kernel. Without `--plan-file` +/// `absent_over_time(data[1m])`: Stages 1-3 plan it, but the +/// Planner's executor has no `absent_over_time` kernel. Without `--plan-file` /// the run succeeds and the document says why the plan is not installable, /// as before. With the compile check (the default, or `compile`) and /// `--plan-file`, the stage output is still written, the run exits 1, and @@ -2442,12 +2442,12 @@ fn plan_file_round_trips_with_and_without_the_executor_check() { /// does not compile there. #[test] fn executor_check_skip_writes_a_plan_the_executor_cannot_compile() { - let query = "sum by (job) (mad_over_time(data[1m]))"; + let query = "absent_over_time(data[1m])"; let workload = TempFile::json(&adhoc(&[query], exact(), &["data"])); let stage3 = ["--workload", workload.path(), "--run-until", "stage3"]; // Precondition: an executor-only gap. Once the executor runs - // `mad_over_time`, pick another query it plans but cannot compile. + // `absent_over_time`, pick another query it plans but cannot compile. let document = document_of(&stage3); let views_error = document["stage3_plan_selection"]["materialized_views_error"] .as_str() diff --git a/crates/planner/src/installable/mod.rs b/crates/planner/src/installable/mod.rs index ec0fa6075..d4cb66b3e 100644 --- a/crates/planner/src/installable/mod.rs +++ b/crates/planner/src/installable/mod.rs @@ -322,7 +322,7 @@ impl InstallablePlan { /// [`Self::validate`] checks a plan file (the DAG's validity, one text /// per root, no standalone scalar query, and each view's and query-time /// scan's retention, which Stage 3 set), but not compiled by the - /// Planner's executor. E.g. a plan with a `last_over_time` the executor + /// Planner's executor. E.g. a plan with an `absent_over_time` the executor /// has no kernel for is returned here and refused by [`Self::of_dag`]. pub fn of_dag_without_executor_check( dag: PhysicalASAPDAG, diff --git a/crates/planner/src/installable/tests.rs b/crates/planner/src/installable/tests.rs index cd72fd516..daa319c19 100644 --- a/crates/planner/src/installable/tests.rs +++ b/crates/planner/src/installable/tests.rs @@ -373,8 +373,8 @@ fn a_root_time_shift_is_read_whole_by_the_installer_not_by_the_cost() { assert_eq!(retention[0].lookback_ms(), None); } -/// `sum by (job) (last_over_time(data[1m]))` every 10 s, exact: Stages 1-3 -/// plan it, but the Planner's executor has no `last_over_time` kernel, so +/// `absent_over_time(data[1m])` every 10 s, exact: Stages 1-3 +/// plan it, but the Planner's executor has no `absent_over_time` kernel, so /// its compile check refuses the plan. A runtime other than the Planner's /// executor still gets the plan, validated, from /// [`InstallablePlan::of_dag_without_executor_check`]. @@ -382,7 +382,7 @@ fn a_root_time_shift_is_read_whole_by_the_installer_not_by_the_cost() { async fn a_plan_only_the_executor_cannot_compile_is_built_without_its_check() { let workload: crate::PlanningWorkload = serde_json::from_str( r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "sum by (job) (last_over_time(data[1m]))", + "query": "absent_over_time(data[1m])", "demand": {"fixed_interval_at": {"interval": 10000, "evaluation_phase": 0}}, "requirements": {"accuracy": {"max_error": 0.0, "failure_probability": 0.0}}, "predictability": {"predictable": {"known_at": null}}}]}, @@ -397,10 +397,10 @@ async fn a_plan_only_the_executor_cannot_compile_is_built_without_its_check() { crate::PlanningModels::builtin(), )) .await - .expect("Stages 1-3 plan last_over_time"); - let query_texts = vec!["sum by (job) (last_over_time(data[1m]))".to_owned()]; + .expect("Stages 1-3 plan absent_over_time"); + let query_texts = vec!["absent_over_time(data[1m])".to_owned()]; // Precondition: an executor-only gap. Once the executor runs - // `last_over_time`, pick another query it plans but cannot compile. + // `absent_over_time`, pick another query it plans but cannot compile. let refused = InstallablePlan::of_dag(output.plan.clone(), query_texts.clone()).unwrap_err(); assert!( matches!( From 7c9bbbacc51ec9e21576938abf5ddd75ad2e09c6 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:44:47 +0000 Subject: [PATCH 4/6] test: plan file always written, the Planner's executor check reported (failing first) The approved CLI: --plan-file alone writes the selected plan, also one the Planner's executor cannot run (reported in the document and a stderr note); no selected plan exits 1 and leaves the file at the path alone; summary families come only from --deployment-capabilities. InstallablePlan::of_dag validates without compiling. --- crates/devtools/tests/asap_planner.rs | 315 +++++++++++++++--------- crates/planner/src/installable/tests.rs | 26 +- tools/dag-viewer/test_viewer.py | 17 +- 3 files changed, 225 insertions(+), 133 deletions(-) diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 3a24c6f20..e2d7f48b1 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -2378,12 +2378,15 @@ fn the_default_text_is_the_terse_form_and_verbose_adds_the_details() { } } -// ── Plan file (`--plan-file`, `--executor-check`) ─────────────────────────── +// ── Plan file (`--plan-file`) and the Planner's executor check ────────────── -/// A plan-file path holding a plan file an earlier run left there: a run -/// overwrites it with its plan, or removes it when it writes none. -fn stale_plan_file() -> TempFile { - TempFile::new("a plan file an earlier run left") +/// What a plan-file path holds before a run: a file an earlier run left. +const EARLIER_PLAN_FILE: &str = "a plan file an earlier run left"; + +/// A plan-file path holding [`EARLIER_PLAN_FILE`]: a run that selects a +/// plan overwrites it; any other run leaves it as it is. +fn earlier_plan_file() -> TempFile { + TempFile::new(EARLIER_PLAN_FILE) } /// The plan file at `path`, read back as a runtime reads it and validated. @@ -2406,14 +2409,27 @@ fn is_the_selected_plan(plan: &asap_planner::InstallablePlan, document: &Value) ) == document["stage3_plan_selection"]["plan"] } +/// The note `asap_planner` writes on standard error when the Planner's +/// executor cannot run the selected plan. +const EXECUTOR_CANNOT_RUN_NOTE: &str = "note: the Planner's executor cannot run this plan: "; + +/// The ready-to-use capabilities file that supports every summary family +/// with every readout it answers. +fn every_summary_family_file() -> String { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tools/dag-viewer/deployment-capabilities/every-summary-family.json") + .to_str() + .unwrap() + .to_owned() +} + /// `--plan-file` writes the selected plan of the committed `example1` -/// workload as -/// `InstallablePlan` JSON, with the executor's compile check (the default) -/// and without it (`skip`): the file reads back with +/// workload as `InstallablePlan` JSON: the file reads back with /// `InstallablePlan::from_json`, validates, holds the workload's query -/// texts in order, and its DAG is the plan the document selected. +/// texts in order, and its DAG is the plan the document selected. The +/// Planner's executor runs it: the document says so and there is no note. #[test] -fn plan_file_round_trips_with_and_without_the_executor_check() { +fn plan_file_round_trips_the_selected_plan() { let workload: PlanningWorkload = serde_json::from_str(&std::fs::read_to_string(example("example1")).unwrap()).unwrap(); let query_texts: Vec = workload @@ -2421,103 +2437,118 @@ fn plan_file_round_trips_with_and_without_the_executor_check() { .entries() .map(|entry| entry.query.0.clone()) .collect(); - for check in [&[][..], &["--executor-check", "skip"]] { - let file = stale_plan_file(); - let mut args = vec!["--run-until", "stage3", "--plan-file", file.path()]; - args.extend_from_slice(check); - let document = example_document("example1", &args); - let plan = read_plan_file(file.path()); - assert_eq!(plan.query_texts, query_texts, "{check:?}"); - assert!(is_the_selected_plan(&plan, &document), "{check:?}"); - } + let file = earlier_plan_file(); + let path = example("example1"); + let args = [ + "--workload", + path.as_str(), + "--run-until", + "stage3", + "--plan-file", + file.path(), + "--format", + "json", + ]; + let output = planner(&args); + assert_eq!(output.status.code(), Some(0)); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(!stderr.contains(EXECUTOR_CANNOT_RUN_NOTE), "{stderr}"); + let document: Value = serde_json::from_slice(&output.stdout).unwrap(); + let plan = read_plan_file(file.path()); + assert_eq!(plan.query_texts, query_texts); + assert!(is_the_selected_plan(&plan, &document)); + let selection = &document["stage3_plan_selection"]; + assert_eq!(selection["planner_executor_can_run"], json!(true)); + assert!(selection["planner_executor_error"].is_null(), "{selection}"); } -/// `absent_over_time(data[1m])`: Stages 1-3 plan it, but the -/// Planner's executor has no `absent_over_time` kernel. Without `--plan-file` -/// the run succeeds and the document says why the plan is not installable, -/// as before. With the compile check (the default, or `compile`) and -/// `--plan-file`, the stage output is still written, the run exits 1, and -/// no plan file is left at the path; with `skip` the plan file is written -/// for a runtime other than the Planner's executor, and that plan really -/// does not compile there. +/// `absent_over_time(data[1m])`: Stages 1-3 plan it, but the Planner's +/// executor has no `absent_over_time` kernel. The plan file is still +/// written and the run exits 0; the executor's gap is only reported: in +/// the document (`planner_executor_can_run: false` with its reason) and in +/// one note on standard error. The document lists the plan's views. #[test] -fn executor_check_skip_writes_a_plan_the_executor_cannot_compile() { +fn a_plan_the_executor_cannot_run_is_written_with_a_note() { let query = "absent_over_time(data[1m])"; let workload = TempFile::json(&adhoc(&[query], exact(), &["data"])); - let stage3 = ["--workload", workload.path(), "--run-until", "stage3"]; - + let file = earlier_plan_file(); + let output = planner(&[ + "--workload", + workload.path(), + "--run-until", + "stage3", + "--plan-file", + file.path(), + "--format", + "json", + ]); + assert_eq!(output.status.code(), Some(0)); + let stderr = String::from_utf8(output.stderr).unwrap(); + let notes: Vec<&str> = stderr + .lines() + .filter(|line| line.starts_with(EXECUTOR_CANNOT_RUN_NOTE)) + .collect(); + assert_eq!(notes.len(), 1, "{stderr}"); + let document: Value = serde_json::from_slice(&output.stdout).unwrap(); + let selection = &document["stage3_plan_selection"]; // Precondition: an executor-only gap. Once the executor runs // `absent_over_time`, pick another query it plans but cannot compile. - let document = document_of(&stage3); - let views_error = document["stage3_plan_selection"]["materialized_views_error"] - .as_str() - .unwrap_or_default(); - assert!(views_error.contains("the executor cannot"), "{document}"); - - for check in [&[][..], &["--executor-check", "compile"]] { - let file = stale_plan_file(); - let mut args = stage3.to_vec(); - args.extend(["--plan-file", file.path()]); - args.extend_from_slice(check); - let output = planner(&args); - assert_eq!(output.status.code(), Some(1), "{check:?}"); - assert!(!output.stdout.is_empty(), "the stage output is written"); - let stderr = String::from_utf8(output.stderr).unwrap(); - assert!( - stderr.contains("the executor cannot"), - "{check:?}: {stderr}" - ); - assert!(!stderr.contains("usage:"), "{stderr}"); - assert!(!Path::new(file.path()).exists(), "{check:?}"); - } - - let file = stale_plan_file(); - let mut args = stage3.to_vec(); - args.extend(["--plan-file", file.path(), "--executor-check", "skip"]); - let document = document_of(&args); + assert_eq!(selection["planner_executor_can_run"], json!(false)); + let reason = selection["planner_executor_error"].as_str().unwrap(); + assert!(reason.contains("AbsentOverTime"), "{reason}"); + assert_eq!(notes[0], format!("{EXECUTOR_CANNOT_RUN_NOTE}{reason}")); + assert!(selection["materialized_views"].is_array(), "{selection}"); + assert!(selection["materialized_views_error"].is_null(), "{selection}"); let plan = read_plan_file(file.path()); assert_eq!(plan.query_texts, [query]); assert!(is_the_selected_plan(&plan, &document)); - assert!(plan.check_compiles().is_err()); - // The document derives the same plan's views, without the compile check. - let selection = &document["stage3_plan_selection"]; - assert!( - selection["materialized_views_error"].is_null(), - "{selection}" - ); - assert!(selection["materialized_views"].is_array(), "{selection}"); + assert_eq!(plan.check_compiles().unwrap_err(), reason); } -/// The deployment capabilities `asap_planner` plans with (the document -/// shows the value it hands to planning): supported summaries the -/// deployment leaves unrestricted are the executor's families under the -/// compile check (the default, as `plan_installable` defaults them) and -/// stay unrestricted with `skip`, so the capabilities the caller passes are -/// the only restriction. +/// When Stage 3 selects no plan (example 3b under cost model 2, which plan +/// selection does not implement), the run exits 1, writes no plan file, +/// says so, and leaves the file already at the path as it is. #[test] -fn executor_check_skip_keeps_unrestricted_summaries_unrestricted() { - let workload = adhoc(&["count by (job) (m)"], within(0.01, 0.01), &["m"]); - let planned_with = |args: &[&str]| { - let mut all = vec!["--run-until", "stage0"]; - all.extend_from_slice(args); - document(&workload, &all)["deployment_capabilities"]["supported_summaries"].clone() - }; - assert_eq!( - planned_with(&["--executor-check", "compile"]), - serde_json::to_value(asap_executor::supported_summary_families()).unwrap() +fn no_selected_plan_exits_1_and_leaves_the_plan_file_path_alone() { + let cost = cost_inputs(&asap_cost::CostInputs { + cost_model: asap_cost::CostModel::ProvisionedCapacity( + asap_cost::ResourceUnitCost::illustrative(), + ), + ..Default::default() + }); + let file = earlier_plan_file(); + let path = example("example3b"); + let output = planner(&[ + "--workload", + path.as_str(), + "--run-until", + "stage3", + "--cost-inputs", + cost.path(), + "--plan-file", + file.path(), + "--format", + "json", + ]); + assert_eq!(output.status.code(), Some(1)); + let document: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!(!selects_a_plan(&document["stage3_plan_selection"])); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains(&format!("no plan file written to {}", file.path())), + "{stderr}" ); assert_eq!( - planned_with(&["--executor-check", "compile"]), - planned_with(&[]) + std::fs::read_to_string(file.path()).unwrap(), + EARLIER_PLAN_FILE ); - assert!(planned_with(&["--executor-check", "skip"]).is_null()); } /// A run that fails before planning (here an unreadable workload file) -/// still removes the plan file an earlier run left at `--plan-file`. +/// leaves the file already at `--plan-file` as it is. #[test] -fn a_failed_run_removes_the_plan_file_left_at_the_path() { - let file = stale_plan_file(); +fn a_failed_run_leaves_the_plan_file_path_alone() { + let file = earlier_plan_file(); refused(&[ "--workload", "/nonexistent/workload.json", @@ -2526,48 +2557,90 @@ fn a_failed_run_removes_the_plan_file_left_at_the_path() { "--plan-file", file.path(), ]); - assert!(!Path::new(file.path()).exists()); + assert_eq!( + std::fs::read_to_string(file.path()).unwrap(), + EARLIER_PLAN_FILE + ); } -/// When Stage 3 selects no plan (example 3b under cost model 2, which plan -/// selection does not implement), `--plan-file` writes none and removes -/// the one an earlier run left, and the run exits 1. +/// The summary families `asap_planner` plans with (the document shows the +/// capabilities it hands to planning) come from `--deployment-capabilities` +/// alone. Without the file, or with a file that does not list +/// `supported_summaries`, they are the Planner's executor's families, as +/// `plan_installable` defaults them; a file that lists them is used as it +/// is: here the ready-to-use file with every summary family, and a file +/// with KLL only. #[test] -fn plan_file_is_not_left_when_no_plan_is_selected() { - let cost = cost_inputs(&asap_cost::CostInputs { - cost_model: asap_cost::CostModel::ProvisionedCapacity( - asap_cost::ResourceUnitCost::illustrative(), - ), - ..Default::default() - }); - let file = stale_plan_file(); - let document = example_document( - "example3b", - &[ - "--run-until", - "stage3", - "--cost-inputs", - cost.path(), - "--plan-file", - file.path(), - ], +fn summary_families_come_from_the_deployment_capabilities() { + let workload = adhoc(&["count by (job) (m)"], within(0.01, 0.01), &["m"]); + let planned_with = |args: &[&str]| { + let mut all = vec!["--run-until", "stage0"]; + all.extend_from_slice(args); + document(&workload, &all)["deployment_capabilities"]["supported_summaries"].clone() + }; + let executor_families = + serde_json::to_value(asap_executor::supported_summary_families()).unwrap(); + assert_eq!(planned_with(&[]), executor_families); + let without_summaries = TempFile::json(&json!({"raw_data_retained": true})); + assert_eq!( + planned_with(&["--deployment-capabilities", without_summaries.path()]), + executor_families + ); + let every_family = every_summary_family_file(); + let declared: Value = + serde_json::from_str(&std::fs::read_to_string(&every_family).unwrap()).unwrap(); + assert_eq!( + planned_with(&["--deployment-capabilities", &every_family]), + declared["supported_summaries"] + ); + let kll = json!([{"summary_type": "Kll", "readouts": ["Quantile"]}]); + let kll_only = TempFile::json(&json!({ "supported_summaries": kll })); + assert_eq!( + planned_with(&["--deployment-capabilities", kll_only.path()]), + kll ); - assert!(!selects_a_plan(&document["stage3_plan_selection"])); - assert!(!Path::new(file.path()).exists()); } -/// An unknown `--executor-check` value, `--plan-file` without its path, and -/// `--plan-file` before Stage 3 selects a plan are usage errors (status 2, +/// `tools/dag-viewer/deployment-capabilities/every-summary-family.json` +/// lists every sketch algorithm with every readout its capability table +/// row answers, so its `supported_summaries` rule out no summary, and +/// nothing else in it rules anything out. +#[test] +fn the_every_summary_family_file_rules_out_nothing() { + use asap_types::deployment_capabilities::DeploymentCapabilities; + use asap_types::summary_type::{SketchAlgorithm, SupportedSummary}; + let capabilities: DeploymentCapabilities = + serde_json::from_str(&std::fs::read_to_string(every_summary_family_file()).unwrap()) + .unwrap(); + let every_family: Vec = SketchAlgorithm::ALL + .iter() + .map(|&algorithm| SupportedSummary { + summary_type: algorithm, + readouts: algorithm + .capability() + .readouts + .iter() + .map(|&(statistic, _)| statistic) + .collect(), + }) + .collect(); + assert_eq!( + capabilities.supported_summaries.as_ref(), + Some(&every_family), + "expected:\n{}", + serde_json::to_string_pretty(&every_family).unwrap() + ); + assert!(capabilities.exclusions().is_empty()); +} + +/// `--plan-file` without its path, `--plan-file` before Stage 3 selects a +/// plan, and the removed `--executor-check` are usage errors (status 2, /// with the usage line) that leave the plan-file path alone. #[test] fn plan_file_flag_misuse_is_a_usage_error() { let workload = TempFile::json(&adhoc(&["sum(data)"], exact(), &["data"])); - let file = stale_plan_file(); + let file = earlier_plan_file(); for (args, message) in [ - ( - vec!["--run-until", "stage3", "--executor-check", "none"], - "unknown executor check none", - ), ( vec!["--run-until", "stage3", "--plan-file"], "--plan-file needs a value", @@ -2576,6 +2649,10 @@ fn plan_file_flag_misuse_is_a_usage_error() { vec!["--run-until", "stage2", "--plan-file", file.path()], "--plan-file needs --run-until stage3", ), + ( + vec!["--run-until", "stage3", "--executor-check", "skip"], + "unknown argument --executor-check", + ), ] { let mut all = vec!["--workload", workload.path()]; all.extend(args); @@ -2587,6 +2664,6 @@ fn plan_file_flag_misuse_is_a_usage_error() { } assert_eq!( std::fs::read_to_string(file.path()).unwrap(), - "a plan file an earlier run left" + EARLIER_PLAN_FILE ); } diff --git a/crates/planner/src/installable/tests.rs b/crates/planner/src/installable/tests.rs index daa319c19..b58129889 100644 --- a/crates/planner/src/installable/tests.rs +++ b/crates/planner/src/installable/tests.rs @@ -4,7 +4,7 @@ use std::rc::Rc; thread_local! { /// The plan nodes handed to the executor's compiler by the views' - /// refreshes in [`InstallablePlan::of_dag`] and + /// refreshes in [`installable_plan_of`] and /// [`InstallablePlan::check_compiles`], summed over their compiles. pub(super) static COMPILED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; } @@ -49,6 +49,7 @@ async fn view_refreshes_compile_only_the_nodes_below_them() { COMPILED_NODES.with(|count| count.set(0)); let rebuilt = InstallablePlan::of_dag(plan.dag.clone(), plan.query_texts.clone()).unwrap(); assert_eq!(rebuilt.materialized_views().unwrap().len(), views); + rebuilt.check_compiles().unwrap(); plan.check_compiles().unwrap(); let compiled = COMPILED_NODES.with(std::cell::Cell::get); assert!( @@ -374,12 +375,12 @@ fn a_root_time_shift_is_read_whole_by_the_installer_not_by_the_cost() { } /// `absent_over_time(data[1m])` every 10 s, exact: Stages 1-3 -/// plan it, but the Planner's executor has no `absent_over_time` kernel, so -/// its compile check refuses the plan. A runtime other than the Planner's -/// executor still gets the plan, validated, from -/// [`InstallablePlan::of_dag_without_executor_check`]. +/// plan it, but the Planner's executor has no `absent_over_time` kernel. +/// [`InstallablePlan::of_dag`] still returns the plan, validated, for any +/// runtime; [`InstallablePlan::check_compiles`] reports the executor's gap, +/// and [`installable_plan_of`] (what `plan_installable` returns) refuses it. #[tokio::test] -async fn a_plan_only_the_executor_cannot_compile_is_built_without_its_check() { +async fn of_dag_validates_a_plan_the_executor_cannot_compile_without_refusing_it() { let workload: crate::PlanningWorkload = serde_json::from_str( r#"{"query_workload": {"language": "promql", "repeating_queries": [{ "query": "absent_over_time(data[1m])", @@ -399,9 +400,14 @@ async fn a_plan_only_the_executor_cannot_compile_is_built_without_its_check() { .await .expect("Stages 1-3 plan absent_over_time"); let query_texts = vec!["absent_over_time(data[1m])".to_owned()]; + let plan = InstallablePlan::of_dag(output.plan.clone(), query_texts) + .expect("validated without the executor's compile check"); + plan.validate().unwrap(); // Precondition: an executor-only gap. Once the executor runs // `absent_over_time`, pick another query it plans but cannot compile. - let refused = InstallablePlan::of_dag(output.plan.clone(), query_texts.clone()).unwrap_err(); + assert!(plan.check_compiles().is_err()); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let refused = installable_plan_of(output.plan.clone(), &entries).unwrap_err(); assert!( matches!( refused, @@ -410,13 +416,9 @@ async fn a_plan_only_the_executor_cannot_compile_is_built_without_its_check() { ), "{refused}" ); - let plan = InstallablePlan::of_dag_without_executor_check(output.plan.clone(), query_texts) - .expect("validated without the executor's compile check"); - plan.validate().unwrap(); - assert!(plan.check_compiles().is_err()); // Still validated: one query text per root. assert!(matches!( - InstallablePlan::of_dag_without_executor_check(output.plan, Vec::new()), + InstallablePlan::of_dag(output.plan, Vec::new()), Err(InstallablePlanError::QueriesDoNotMatchRoots { roots: 1 }) )); } diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index f60d85c20..96bf70c48 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -801,10 +801,10 @@ def test_a_document_fills_the_data_workload_and_says_why_the_plan_has_no_views(s """A planned workload fills the data workload and deployment panels; a plan that is not installable says why under materialized views.""" doc = example1() - # Example 1's plan installs; a plan the executor cannot compile has + # Example 1's plan installs; a plan whose views cannot be derived has # this error instead of its views. doc["stage3_plan_selection"].pop("materialized_views") - doc["stage3_plan_selection"]["materialized_views_error"] = "the executor cannot compile the plan's query run" + doc["stage3_plan_selection"]["materialized_views_error"] = "materialized view 3 reads a view refreshed later" js = self.page("?doc=s.json", {"s.json": json.dumps(doc)}) self.assertIn("q1, q2", self.text(js, "dataSources")) self.assertIn("ingesting", js.eval("__els.dataArrival.textContent")) @@ -812,6 +812,19 @@ def test_a_document_fills_the_data_workload_and_says_why_the_plan_has_no_views(s self.assertIn("every 10 s · predictable", self.text(js, "queries")) self.assertIn(f"no installable plan: {doc['stage3_plan_selection']['materialized_views_error']}", self.text(js, "views-meta")) + def test_a_plan_the_planners_executor_cannot_run_lists_its_views_and_says_why(self): + """A plan the Planner's executor cannot run still lists its views; + the views line says why the executor cannot run it.""" + doc = example1() + doc["stage3_plan_selection"].pop("materialized_views_error", None) + doc["stage3_plan_selection"]["materialized_views"] = [materialized_view(2)] + doc["stage3_plan_selection"]["planner_executor_can_run"] = False + doc["stage3_plan_selection"]["planner_executor_error"] = "AbsentOverTime has no native implementation" + js = self.page("?doc=s.json", {"s.json": json.dumps(doc)}) + self.assertIn("1 materialized view", self.text(js, "views-meta")) + self.assertIn("the Planner's executor cannot run this plan: AbsentOverTime has no native implementation", + self.text(js, "views-meta")) + def test_materialized_views_table_has_one_row_per_view_and_a_row_selects_its_node(self): """The materialized views table has one row per view; clicking it shows its plan node.""" From dc1a2b7a433135b1957ad0d8c6d0d533574b5dbf Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:44:50 +0000 Subject: [PATCH 5/6] feat: report the Planner's executor check instead of gating the plan file Delete --executor-check. --plan-file writes the selected plan whether or not the Planner's executor can run it; the compile check's result is the document's planner_executor_can_run / planner_executor_error and one stderr note. No selected plan: exit 1, no file, an existing file untouched. Summary families come only from --deployment-capabilities (default: the executor's); every-summary-family.json is the ready-to-use unrestricted file. InstallablePlan::of_dag is the one public constructor (validated, not compiled); installable_plan_of adds the executor's compile check. --- crates/devtools/src/bin/asap_planner.rs | 121 +++++++----------- crates/devtools/tests/asap_planner.rs | 23 +++- crates/devtools/tests/corpus_snapshots.rs | 7 +- .../corpus_snapshots/o11y_bench/q05.json | 75 ++++++++++- .../corpus_snapshots/o11y_bench/q06.json | 40 +++++- .../corpus_snapshots/o11y_bench/q11.json | 2 +- crates/planner/src/installable/mod.rs | 42 +++--- docs/user_guide_docs/asap-planner.md | 51 +++++++- tools/dag-viewer/README.md | 4 +- tools/dag-viewer/app.js | 5 +- .../every-summary-family.json | 32 +++++ tools/validate_joint_workloads.py | 2 + 12 files changed, 288 insertions(+), 116 deletions(-) create mode 100644 tools/dag-viewer/deployment-capabilities/every-summary-family.json diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 97c9d6efb..340934803 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -44,29 +44,29 @@ // `query_time_results_retained`, `counter_semantics`, // `state_not_mergeable_across_windows`, `excluded_candidates`; an // omitted field takes its default). -// Without `supported_summaries`, the executor's summary families are -// supported, as `plan_installable` defaults it, unless -// `--executor-check skip`. +// Without `supported_summaries` (or without the file), the Planner's +// executor's summary families are supported, as `plan_installable` +// defaults it. To plan for a runtime that builds every summary family, +// pass a file listing them all, e.g. +// `tools/dag-viewer/deployment-capabilities/every-summary-family.json`. // - `--ignore-accuracy-guarantees`: Stage 3 accepts every query output // regardless of its accuracy guarantee // (`PlanningModels::ignore_accuracy_guarantees`), comparing candidates // on cost alone. By default it enforces them, as `plan_installable` // does. -// - `--executor-check compile|skip`: whether the selected plan must -// compile on the Planner's executor (`compile`, the default, as -// `plan_installable` checks it), or is for a runtime other than the -// Planner's executor (`skip`): then unrestricted `supported_summaries` -// stay unrestricted, so the deployment capabilities passed are the only -// restriction, and the plan is validated -// (`InstallablePlan::of_dag_without_executor_check`) but not compiled. +// - `--plan-file ` (needs `--run-until stage3`): besides the stage +// output, writes the selected plan as an `InstallablePlan` JSON plan +// file (`InstallablePlan::of_dag`), which a runtime reads with +// `InstallablePlan::from_json`. When Stage 3 selects no plan, no file is +// written (a file already at the path is left as it is) and the run +// exits 1 after writing the stage output. // -// Besides the stage output, `--plan-file ` (with `--run-until -// stage3`) writes the selected plan as an `InstallablePlan` JSON plan file, -// which a runtime reads with `InstallablePlan::from_json`, checked as -// `--executor-check` says. When no plan is selected or the plan fails the -// check, no plan file is written and the run exits 1 after writing the -// stage output. A plan file left at the path is removed once the arguments -// are read, so no failed run leaves one there. +// With a selected plan, the run also compiles it on the Planner's executor +// (`InstallablePlan::check_compiles`) and reports the result: the document's +// `planner_executor_can_run`, and when the executor cannot run the plan a +// line `note: the Planner's executor cannot run this plan: ` on +// standard error. The result never blocks the plan file or the exit status: +// a runtime other than the Planner's executor may run the plan. // // The document (`--format json`): // - workload: the queries (`id`, `language`, `text`, `requirements`, @@ -98,8 +98,9 @@ // `parameters`, `timing`, `lifetime`) and the `queries` reading it. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them; `materialized_views_error` when the -// plan is not installable (under `--executor-check skip`, not checked -// against the Planner's executor). `warnings`: a PromQL query applying `rate`, `increase` +// plan is not installable. `planner_executor_can_run`: whether the +// Planner's executor compiles the plan, with `planner_executor_error`, +// its reason, when it does not. `warnings`: a PromQL query applying `rate`, `increase` // or `irate` to a declared gauge or combining a declared summary's // quantile series across series (`metric_type_warnings`), each data fact the selected // plan assumed because the data workload does not declare it (`assumed_data_facts`, @@ -177,7 +178,7 @@ use serde_json::{json, Value}; const USAGE: &str = "usage: asap_planner (--workload )... \ --run-until stage0|stage1_pass1|stage1_pass2|stage2|stage3 [--format text|json] [--verbose] [--out ] \ [--sql-catalog-table ]... [--cost-inputs ] [--deployment-capabilities ] \ -[--ignore-accuracy-guarantees] [--executor-check compile|skip] [--plan-file ]"; +[--ignore-accuracy-guarantees] [--plan-file ]"; fn main() { match run(std::env::args().skip(1).collect()) { @@ -206,17 +207,6 @@ impl From for Failure { } } -/// Whether the selected plan must compile on the Planner's executor -/// (`--executor-check`). -#[derive(Clone, Copy, PartialEq, Eq)] -enum ExecutorCheck { - /// It must, as `plan_installable` checks it (the default). - Compile, - /// It is for a runtime other than the Planner's executor: validated, - /// not compiled, and planned with the summaries the deployment declares. - Skip, -} - /// `path`'s contents as `T`, errors naming `flag` and `path`. fn read_json(flag: &str, path: &str) -> Result { let text = std::fs::read_to_string(path).map_err(|e| format!("{flag} {path}: {e}"))?; @@ -233,7 +223,6 @@ fn run(args: Vec) -> Result<(), Failure> { let mut deployment_capabilities = DeploymentCapabilities::default(); let mut ignore_accuracy_guarantees = false; let mut verbose = false; - let mut executor_check = ExecutorCheck::Compile; let mut plan_file = None; let mut args = args.into_iter(); while let Some(flag) = args.next() { @@ -264,13 +253,6 @@ fn run(args: Vec) -> Result<(), Failure> { "--printing-full-candidates was renamed to --verbose".into(), )) } - "--executor-check" => { - executor_check = match value()?.as_str() { - "compile" => ExecutorCheck::Compile, - "skip" => ExecutorCheck::Skip, - other => return Err(Failure::Usage(format!("unknown executor check {other}"))), - } - } "--plan-file" => plan_file = Some(value()?), other => return Err(Failure::Usage(format!("unknown argument {other}"))), } @@ -284,17 +266,6 @@ fn run(args: Vec) -> Result<(), Failure> { if workload_paths.is_empty() { return Err(Failure::Usage("--workload is required".into())); } - // A plan file an earlier run left at the path is removed first, so a - // runtime never installs a plan this run did not select, whichever way - // the run fails. - if let Some(path) = &plan_file { - match std::fs::remove_file(path) { - Err(e) if e.kind() != std::io::ErrorKind::NotFound => { - return Err(format!("--plan-file {path}: {e}").into()) - } - _ => {} - } - } let mut workloads = Vec::new(); for path in &workload_paths { let workload: PlanningWorkload = read_json("--workload", path)?; @@ -306,10 +277,8 @@ fn run(args: Vec) -> Result<(), Failure> { let (catalog, table_names) = sql_catalog(&tables)?; let (described, roots, entries, data) = lowered_together(&workloads, &catalog, &table_names)?; // Plan with the executor's summary families when the deployment declares - // none, unless the plan is for a runtime other than the Planner's executor. - if executor_check == ExecutorCheck::Compile - && deployment_capabilities.supported_summaries.is_none() - { + // none. + if deployment_capabilities.supported_summaries.is_none() { deployment_capabilities.supported_summaries = Some(asap_executor::supported_summary_families()); } @@ -325,12 +294,20 @@ fn run(args: Vec) -> Result<(), Failure> { .and_then(|run| run.run.selection.as_ref().ok()) .and_then(|(plan, _)| asap_cost::plan_cost(plan, &entries, &data, &cost_inputs).ok()) .map(|(cost, _)| cost); - // The selected plan as installed, checked as `--executor-check` says. + // The selected plan as a runtime installs it, validated. let installable = result .as_ref() .ok() .and_then(|run| run.run.selection.as_ref().ok()) - .map(|(plan, _)| installable_plan(plan, &entries, executor_check)); + .map(|(plan, _)| { + let query_texts = entries.iter().map(|entry| entry.query.0.clone()).collect(); + InstallablePlan::of_dag(plan.clone(), query_texts).map_err(|e| e.to_string()) + }); + // Whether the Planner's executor compiles it: reported, never blocking. + let executor_can_run = installable + .as_ref() + .and_then(|installable| installable.as_ref().ok()) + .map(InstallablePlan::check_compiles); // Its materialized views, as `plan_installable` derives them. let materialized_views = installable.as_ref().map(|installable| { let plan = installable.as_ref().map_err(Clone::clone)?; @@ -372,6 +349,12 @@ fn run(args: Vec) -> Result<(), Failure> { )?; // Stage 3 is null when Pass 1 rejected a query. if let Some(stage3) = document["stage3_plan_selection"].as_object_mut() { + if let Some(can_run) = &executor_can_run { + stage3.insert("planner_executor_can_run".into(), json!(can_run.is_ok())); + if let Err(reason) = can_run { + stage3.insert("planner_executor_error".into(), json!(reason)); + } + } if !warnings.is_empty() { stage3.insert("warnings".into(), json!(warnings)); } @@ -417,6 +400,9 @@ fn run(args: Vec) -> Result<(), Failure> { Some(path) => std::fs::write(&path, text).map_err(|e| format!("{path}: {e}"))?, None => print!("{text}"), } + if let Some(Err(reason)) = &executor_can_run { + eprintln!("note: the Planner's executor cannot run this plan: {reason}"); + } if let Some(path) = plan_file { match installable { Some(Ok(plan)) => { @@ -425,7 +411,9 @@ fn run(args: Vec) -> Result<(), Failure> { } Some(Err(error)) => failure = Some(format!("--plan-file {path}: {error}")), // No plan was selected: `failure` already says why. - None => {} + None => { + failure = failure.map(|error| format!("{error} (no plan file written to {path})")) + } } } match failure { @@ -775,25 +763,6 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option Result { - let query_texts = entries.iter().map(|entry| entry.query.0.clone()).collect(); - let plan = InstallablePlan::of_dag_without_executor_check(plan.clone(), query_texts) - .map_err(|e| e.to_string())?; - if executor_check == ExecutorCheck::Compile { - plan.check_compiles()?; - } - Ok(plan) -} - /// Stage 2's physical candidates $\mathcal C$: each with its logical /// candidate (`group`, `index_in_group`), its `properties` ($\tau$, /// $\ell$) and, per input $(c, i)$, the input `group` and, as `edges`, diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index e2d7f48b1..fc0c8aaa9 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -2498,7 +2498,10 @@ fn a_plan_the_executor_cannot_run_is_written_with_a_note() { assert!(reason.contains("AbsentOverTime"), "{reason}"); assert_eq!(notes[0], format!("{EXECUTOR_CANNOT_RUN_NOTE}{reason}")); assert!(selection["materialized_views"].is_array(), "{selection}"); - assert!(selection["materialized_views_error"].is_null(), "{selection}"); + assert!( + selection["materialized_views_error"].is_null(), + "{selection}" + ); let plan = read_plan_file(file.path()); assert_eq!(plan.query_texts, [query]); assert!(is_the_selected_plan(&plan, &document)); @@ -2603,11 +2606,11 @@ fn summary_families_come_from_the_deployment_capabilities() { /// `tools/dag-viewer/deployment-capabilities/every-summary-family.json` /// lists every sketch algorithm with every readout its capability table -/// row answers, so its `supported_summaries` rule out no summary, and -/// nothing else in it rules anything out. +/// row answers, so its `supported_summaries` rule out no summary type and +/// no readout; its other fields keep their defaults. #[test] fn the_every_summary_family_file_rules_out_nothing() { - use asap_types::deployment_capabilities::DeploymentCapabilities; + use asap_types::deployment_capabilities::{DeploymentCapabilities, ExclusionSource}; use asap_types::summary_type::{SketchAlgorithm, SupportedSummary}; let capabilities: DeploymentCapabilities = serde_json::from_str(&std::fs::read_to_string(every_summary_family_file()).unwrap()) @@ -2630,7 +2633,17 @@ fn the_every_summary_family_file_rules_out_nothing() { "expected:\n{}", serde_json::to_string_pretty(&every_family).unwrap() ); - assert!(capabilities.exclusions().is_empty()); + let summary_exclusions: Vec<_> = capabilities + .exclusions() + .into_iter() + .filter(|exclusion| { + matches!( + exclusion.source, + ExclusionSource::SummaryNotSupported | ExclusionSource::ReadoutNotSupported + ) + }) + .collect(); + assert!(summary_exclusions.is_empty(), "{summary_exclusions:?}"); } /// `--plan-file` without its path, `--plan-file` before Stage 3 selects a diff --git a/crates/devtools/tests/corpus_snapshots.rs b/crates/devtools/tests/corpus_snapshots.rs index 8f163b991..b340ab8e8 100644 --- a/crates/devtools/tests/corpus_snapshots.rs +++ b/crates/devtools/tests/corpus_snapshots.rs @@ -122,7 +122,12 @@ fn plan_snapshot(document: &Value) -> Map { "accepted": stage3["accepted_accuracy_per_query"], }), ); - for key in ["materialized_views", "materialized_views_error", "warnings"] { + for key in [ + "materialized_views", + "materialized_views_error", + "planner_executor_error", + "warnings", + ] { if let Some(value) = stage3.get(key) { snapshot.insert(key.into(), value.clone()); } diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json index 621527658..a0535ab49 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json @@ -1,7 +1,78 @@ { "accuracy_guarantees": "enforced", - "materialized_views": [], - "materialized_views_error": "the executor cannot compile the plan's query run: invalid DAG: node 13: invalid DAG: value operation has no native implementation", + "materialized_views": [ + { + "coverage": { + "definition_id": "7e48d8927e84a32d499eaaf08711a04d", + "selection": [ + { + "columns": [ + { + "column": { + "column": "job", + "scan_ordinal": 0, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "constraint": { + "in": [ + { + "Utf8": "order-service" + } + ] + } + } + ] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 1, + "refresh": "at_ingestion", + "retention_ms": 3900000 + }, + { + "coverage": { + "definition_id": "58395555aae268f4f68a8a3e82a6c45b", + "selection": [ + { + "columns": [ + { + "column": { + "column": "job", + "scan_ordinal": 0, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "constraint": { + "in": [ + { + "Utf8": "order-service" + } + ] + } + } + ] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 7, + "refresh": "at_ingestion", + "retention_ms": 3900000 + } + ], + "planner_executor_error": "the executor cannot compile the plan's query run: invalid DAG: node 13: invalid DAG: value operation has no native implementation", "query": "max_over_time((sum(rate(http_requests_total{job=\"order-service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"order-service\"}[5m])))[6h:1m])", "selected": { "accepted": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json index 1f26986f6..af918f144 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json @@ -1,7 +1,43 @@ { "accuracy_guarantees": "enforced", - "materialized_views": [], - "materialized_views_error": "the executor cannot compile the plan's query run: invalid DAG: node 7: invalid DAG: value operation has no native implementation", + "materialized_views": [ + { + "coverage": { + "definition_id": "be9c29c7ba723e7271224b1879a6c7b4", + "selection": [ + { + "columns": [ + { + "column": { + "column": "job", + "scan_ordinal": 0, + "source": { + "TimeSeries": { + "metric": "http_request_duration_seconds_bucket" + } + } + }, + "constraint": { + "in": [ + { + "Utf8": "user-service" + } + ] + } + } + ] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 1, + "refresh": "at_ingestion", + "retention_ms": 3900000 + } + ], + "planner_executor_error": "the executor cannot compile the plan's query run: invalid DAG: node 7: invalid DAG: value operation has no native implementation", "query": "max_over_time(histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"user-service\"}[5m])) by (le))[12h:1m])", "selected": { "accepted": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json index 779c7b5d5..dca914613 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json @@ -1,7 +1,7 @@ { "accuracy_guarantees": "enforced", "materialized_views": [], - "materialized_views_error": "the executor cannot compile the plan's query run: invalid DAG: node 3: invalid DAG: value operation has no native implementation", + "planner_executor_error": "the executor cannot compile the plan's query run: invalid DAG: node 3: invalid DAG: value operation has no native implementation", "query": "max_over_time(sum(process_resident_memory_bytes)[6h:1m])", "selected": { "accepted": [ diff --git a/crates/planner/src/installable/mod.rs b/crates/planner/src/installable/mod.rs index d4cb66b3e..5290f93ad 100644 --- a/crates/planner/src/installable/mod.rs +++ b/crates/planner/src/installable/mod.rs @@ -234,15 +234,18 @@ pub async fn plan_installable( /// The installable plan of `plan`, the plan Stage 3 built for a workload /// whose entries are `entries` (in `QueryWorkload::entries()` order): what -/// [`plan_installable`] returns after planning, without its cost report. A -/// caller that runs the planning stages itself (e.g. a developer tool) uses -/// it to see what a runtime would install. +/// [`plan_installable`] returns after planning, without its cost report. +/// [`InstallablePlan::of_dag`], with each view's refresh and the query run +/// compiled ([`executor_compiles`]): a plan the Planner's executor cannot +/// run is refused here, not installed. pub fn installable_plan_of( plan: PhysicalASAPDAG, entries: &[QueryWorkloadEntry], ) -> Result { let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - InstallablePlan::of_dag(plan, query_texts) + let plan = InstallablePlan::of_dag(plan, query_texts)?; + executor_compiles(&plan.dag, &plan.materialized_views()?)?; + Ok(plan) } impl InstallablePlan { @@ -296,7 +299,8 @@ impl InstallablePlan { /// DAG's own `TimeRange`s are compiled, so a window the executor cannot /// run fails here. `Err` holds the executor's error, or why the views /// cannot be derived. [`plan_installable`] runs the same compiles before - /// it returns a plan; this checks a plan read from a plan file. + /// it returns a plan; this checks a plan read from a plan file or built + /// by [`Self::of_dag`]. pub fn check_compiles(&self) -> Result<(), String> { let views = self .materialized_views() @@ -304,27 +308,15 @@ impl InstallablePlan { executor_compiles(&self.dag, &views).map_err(|error| error.to_string()) } - /// The plan of `dag`, validated ([`Self::of_dag_without_executor_check`]), - /// with each view's refresh and the query run compiled - /// ([`executor_compiles`]): a plan the Planner's executor cannot run is - /// refused here, not installed. - pub(crate) fn of_dag( - dag: PhysicalASAPDAG, - query_texts: Vec, - ) -> Result { - let plan = Self::of_dag_without_executor_check(dag, query_texts)?; - executor_compiles(&plan.dag, &plan.materialized_views()?)?; - Ok(plan) - } - /// The plan of `dag` answering `query_texts` (one per root, in workload - /// order) for a runtime other than the Planner's executor: checked as - /// [`Self::validate`] checks a plan file (the DAG's validity, one text - /// per root, no standalone scalar query, and each view's and query-time - /// scan's retention, which Stage 3 set), but not compiled by the - /// Planner's executor. E.g. a plan with an `absent_over_time` the executor - /// has no kernel for is returned here and refused by [`Self::of_dag`]. - pub fn of_dag_without_executor_check( + /// order), checked as [`Self::validate`] checks a plan file: the DAG's + /// validity, one text per root, no standalone scalar query, and each + /// view's and query-time scan's retention, which Stage 3 set. Not + /// compiled by the Planner's executor ([`Self::check_compiles`] does + /// that), so a runtime other than the Planner's executor gets every plan + /// Stage 3 selects: e.g. a plan with an `absent_over_time`, which the + /// Planner's executor has no kernel for. + pub fn of_dag( dag: PhysicalASAPDAG, query_texts: Vec, ) -> Result { diff --git a/docs/user_guide_docs/asap-planner.md b/docs/user_guide_docs/asap-planner.md index 3ccd5bbb9..fd6cc7f98 100644 --- a/docs/user_guide_docs/asap-planner.md +++ b/docs/user_guide_docs/asap-planner.md @@ -36,7 +36,7 @@ minutes in a debug build. asap_planner (--workload )... --run-until stage0|stage1_pass1|stage1_pass2|stage2|stage3 [--format text|json] [--verbose] [--out ] [--sql-catalog-table ]... [--cost-inputs ] [--deployment-capabilities ] - [--ignore-accuracy-guarantees] + [--ignore-accuracy-guarantees] [--plan-file ] ``` | Option | Meaning | @@ -50,6 +50,7 @@ asap_planner (--workload )... --run-until stage0|stage1_pass1|stage1_pass2 | `--cost-inputs ` | The cost model and its costs (`CostInputs` JSON). The files in `tools/dag-viewer/cost-inputs/` are ready to use. Without it, the built-in usage-over-time model is used. Stage 3 costs only usage-over-time models (`aws-fargate-us-east-1.json` is one); a provisioned- or cluster-capacity model ends Stage 3 with an `UnsupportedCostModel` error. | | `--deployment-capabilities ` | What the deployment can do, e.g. `{"raw_data_retained": true, "query_time_results_retained": false}`. Set it to match your deployment; see [Deployment capabilities](#deployment-capabilities). | | `--ignore-accuracy-guarantees` | Accept candidates without an accuracy guarantee. By default each query's accuracy requirement is enforced. | +| `--plan-file ` | Also write the plan Stage 3 selects to this file, as `InstallablePlan` JSON; needs `--run-until stage3`. See [Write a plan file](#write-a-plan-file). | Any other option, or a missing required one, is a usage error (exit status 2). An input the planner cannot read exits with status 1. So does a @@ -57,10 +58,33 @@ run in which a stage failed (for example, Pass 1 rejected a query, or no plan meets the requirements): the output is still written, with the error under the stage that failed. +With a selected plan, `asap_planner` also compiles it on the Planner's +executor and reports the result; it never changes the exit status. When the +executor cannot run the plan, standard error gets one line + +```text +note: the Planner's executor cannot run this plan: +``` + +and the JSON document's `stage3_plan_selection` has +`"planner_executor_can_run": false` with the reason in +`planner_executor_error` (otherwise `"planner_executor_can_run": true`). + ## Deployment capabilities The `--deployment-capabilities` file is a `DeploymentCapabilities` JSON -object; an omitted field keeps its default. Besides +object; an omitted field keeps its default. + +`supported_summaries` lists the sketch summary types the deployment builds, +each with the statistics it reads from them, e.g. +`[{"summary_type": "Kll", "readouts": ["Quantile"]}]`. Without the file, or +when the file leaves it out, the planner uses the summary types the +Planner's executor builds. To plan for a runtime that builds every summary +type, pass the ready-to-use +[`every-summary-family.json`](../../tools/dag-viewer/deployment-capabilities/every-summary-family.json), +which lists each one with every statistic it answers. + +Besides `supported_summaries`, `raw_data_retained` and `query_time_results_retained`, three fields say what the executor cannot do or what the caller rules out: @@ -141,6 +165,29 @@ cargo run --release -p asap-devtools --bin asap_planner -- \ See [Run and inspect a query](run-a-query.md) for workload, source-fact and SQL examples. To try a single query, write a one-query workload file. +## Write a plan file + +`--plan-file ` writes the selected plan as an `InstallablePlan` JSON +file (`wire_version`, the selected DAG and the query texts), the input a +runtime installs; a runtime reads it with `InstallablePlan::from_json`. The +plan file is written whether or not the Planner's executor can run the plan: +another runtime may run it. + +```sh +cargo run --release -p asap-devtools --bin asap_planner -- \ + --workload tools/dag-viewer/workloads/planner-layering/example1.json \ + --run-until stage3 --plan-file plan.json +``` + +| Run | Exit status | `plan.json` | +| --- | --- | --- | +| Stage 3 selects a plan the Planner's executor runs | 0 | written | +| Stage 3 selects a plan the Planner's executor cannot run | 0, with the `note:` line on standard error | written | +| Stage 3 selects no plan, or the run fails | 1, with `asap_planner: (no plan file written to plan.json)` when Stage 3 selected none | not written; a file already there is left as it is | + +`--plan-file` with a `--run-until` before `stage3` is a usage error (exit +status 2). + ## Inspect the result With `--format json`, open the document in the diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index e2e507fa6..9213bd374 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -151,7 +151,9 @@ whole document): `pruned_candidates` (as Stage 2's), `milp_variables`, `milp_constraints`, `materialized_views` (`node_id`, a plan node, `refresh`, `coverage.definition_id`, `inputs` and `retention_ms`), - `materialized_views_error`, `warnings` and `error`; + `materialized_views_error`, `planner_executor_can_run` (whether the + Planner's executor compiles the plan) with `planner_executor_error` when + it does not, `warnings` and `error`; - `memo.steps[].groups[]`: `id`, `label`, `operator` (the group's expression as `Operator` JSON, children as group ids), `children`, `parents`, `queries`, `roots_of`, `candidate_count`, diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index 4f6a213d9..3cd385cdb 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -227,7 +227,10 @@ $('views-meta').innerHTML = stage3.materialized_views_error ? `no installable plan: ${esc(stage3.materialized_views_error)}` : `${rows.length} materialized view${rows.length === 1 ? '' : 's'}` + - (rows.length ? '' : ': nothing is materialized across evaluations; every query computes at query time'); + (rows.length ? '' : ': nothing is materialized across evaluations; every query computes at query time') + + (stage3.planner_executor_error + ? ` · the Planner's executor cannot run this plan: ${esc(stage3.planner_executor_error)}` + : ''); $('materializedViews').innerHTML = rows.length ? table( ['plan node', 'definition', 'kind', 'refresh', 'reads', 'retention', 'Stage 3 cost', 'read by'], rows.map((r) => `${esc(r.nodeId)}` + diff --git a/tools/dag-viewer/deployment-capabilities/every-summary-family.json b/tools/dag-viewer/deployment-capabilities/every-summary-family.json new file mode 100644 index 000000000..cb8d73a6b --- /dev/null +++ b/tools/dag-viewer/deployment-capabilities/every-summary-family.json @@ -0,0 +1,32 @@ +{ + "supported_summaries": [ + {"summary_type": "Cms", "readouts": ["ItemFrequency", "Total"]}, + {"summary_type": "CountSketch", "readouts": ["ItemFrequency"]}, + {"summary_type": "CountSketchWithL2Norm", "readouts": ["ItemFrequency", "FrequencyL2"]}, + {"summary_type": "CmsWithHeap", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CountSketchWithHeap", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CmsWithSpaceSaving", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CountSketchWithSpaceSaving", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CmsFold", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CountSketchFold", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "Hll", "readouts": ["Distinct"]}, + {"summary_type": "HllHip", "readouts": ["Distinct"]}, + {"summary_type": "Kmv", "readouts": ["Distinct"]}, + {"summary_type": "Kll", "readouts": ["Quantile", "Rank"]}, + {"summary_type": "KllDynamic", "readouts": ["Quantile", "Rank"]}, + {"summary_type": "DDSketch", "readouts": ["Quantile"]}, + {"summary_type": "SpaceSaving", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "Elastic", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "Coco", "readouts": ["ItemFrequency", "PartialKeySum"]}, + {"summary_type": "Bloom", "readouts": ["Membership"]}, + {"summary_type": "SetAggregator", "readouts": ["Distinct", "Membership"]}, + {"summary_type": "UnivMon", "readouts": ["Total", "FrequencyL2", "Distinct", "FrequencyEntropy", "GSum"]}, + {"summary_type": "UnivMonTwoTier", "readouts": ["Total", "FrequencyL2", "Distinct", "FrequencyEntropy", "GSum"]}, + {"summary_type": "UnivMonQ", "readouts": ["ItemFrequency", "FrequencyL2", "Distinct", "Quantile", "Rank", "GSum", "FrequencyEntropy", "TopK"]}, + {"summary_type": "HydraCms", "readouts": ["ItemFrequency", "Total"]}, + {"summary_type": "HydraCountSketch", "readouts": ["ItemFrequency"]}, + {"summary_type": "HydraKll", "readouts": ["Quantile", "Rank"]}, + {"summary_type": "HydraHll", "readouts": ["Distinct"]}, + {"summary_type": "HydraUnivMon", "readouts": ["Total", "Distinct", "FrequencyL2", "FrequencyEntropy"]} + ] +} diff --git a/tools/validate_joint_workloads.py b/tools/validate_joint_workloads.py index 7c9b227e7..dc178a232 100644 --- a/tools/validate_joint_workloads.py +++ b/tools/validate_joint_workloads.py @@ -80,6 +80,7 @@ def run(entry): result['selected'] = selected result['error'] = selection.get('error') or (document.get('stage1_logical_asap_optimization') or {}).get('error') result['materialized_views_error'] = selection.get('materialized_views_error') + result['planner_executor_error'] = selection.get('planner_executor_error') report = selection.get('search_report') or {} result['search'] = report if selected: @@ -110,6 +111,7 @@ def run(entry): and result.get('guarantees_enforced') and result['cost_certificate_valid'] and result.get('accepted_queries') == result['expected_queries'] and not result.get('error') and not result.get('materialized_views_error') + and not result.get('planner_executor_error') ) print(json.dumps({k: result.get(k) for k in ['workload', 'wall_seconds', 'selected', 'accepted_queries', 'modeled_cost_per_second', 'error']}), flush=True) return result From 67f17d5f29542be67845b1dbffa8768c0b0e3a61 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:08:45 +0000 Subject: [PATCH 6/6] refactor: apply the independent review of the plan file CLI A plan file write error names --plan-file; materialized_views_error is documented (and shown in the viewer) as an invalid plan, not an executor gap; the misuse test drops a flag main never had. --- crates/devtools/src/bin/asap_planner.rs | 12 +++++++----- crates/devtools/tests/asap_planner.rs | 10 +++------- tools/dag-viewer/README.md | 3 ++- tools/dag-viewer/app.js | 2 +- tools/dag-viewer/test_viewer.py | 2 +- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 340934803..1441df9ef 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -98,10 +98,11 @@ // `parameters`, `timing`, `lifetime`) and the `queries` reading it. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them; `materialized_views_error` when the -// plan is not installable. `planner_executor_can_run`: whether the -// Planner's executor compiles the plan, with `planner_executor_error`, -// its reason, when it does not. `warnings`: a PromQL query applying `rate`, `increase` -// or `irate` to a declared gauge or combining a declared summary's +// plan fails `InstallablePlan::validate` (its views cannot be derived). +// `planner_executor_can_run`: whether the Planner's executor compiles +// the plan, with `planner_executor_error`, its reason, when it does +// not. `warnings`: a PromQL query applying `rate`, `increase` or +// `irate` to a declared gauge or combining a declared summary's // quantile series across series (`metric_type_warnings`), each data fact the selected // plan assumed because the data workload does not declare it (`assumed_data_facts`, // saying whether answers may then be wrong beyond the accuracy target), and a peak @@ -407,7 +408,8 @@ fn run(args: Vec) -> Result<(), Failure> { match installable { Some(Ok(plan)) => { let json = serde_json::to_string_pretty(&plan).map_err(|e| e.to_string())?; - std::fs::write(&path, json + "\n").map_err(|e| format!("{path}: {e}"))?; + std::fs::write(&path, json + "\n") + .map_err(|e| format!("--plan-file {path}: {e}"))?; } Some(Err(error)) => failure = Some(format!("--plan-file {path}: {error}")), // No plan was selected: `failure` already says why. diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index fc0c8aaa9..482bf6981 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -2646,9 +2646,9 @@ fn the_every_summary_family_file_rules_out_nothing() { assert!(summary_exclusions.is_empty(), "{summary_exclusions:?}"); } -/// `--plan-file` without its path, `--plan-file` before Stage 3 selects a -/// plan, and the removed `--executor-check` are usage errors (status 2, -/// with the usage line) that leave the plan-file path alone. +/// `--plan-file` without its path and `--plan-file` before Stage 3 selects +/// a plan are usage errors (status 2, with the usage line) that leave the +/// plan-file path alone. #[test] fn plan_file_flag_misuse_is_a_usage_error() { let workload = TempFile::json(&adhoc(&["sum(data)"], exact(), &["data"])); @@ -2662,10 +2662,6 @@ fn plan_file_flag_misuse_is_a_usage_error() { vec!["--run-until", "stage2", "--plan-file", file.path()], "--plan-file needs --run-until stage3", ), - ( - vec!["--run-until", "stage3", "--executor-check", "skip"], - "unknown argument --executor-check", - ), ] { let mut all = vec!["--workload", workload.path()]; all.extend(args); diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index 9213bd374..4ed08f55a 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -151,7 +151,8 @@ whole document): `pruned_candidates` (as Stage 2's), `milp_variables`, `milp_constraints`, `materialized_views` (`node_id`, a plan node, `refresh`, `coverage.definition_id`, `inputs` and `retention_ms`), - `materialized_views_error`, `planner_executor_can_run` (whether the + `materialized_views_error` (the plan is invalid: its views cannot be + derived), `planner_executor_can_run` (whether the Planner's executor compiles the plan) with `planner_executor_error` when it does not, `warnings` and `error`; - `memo.steps[].groups[]`: `id`, `label`, `operator` (the group's diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index 3cd385cdb..54e4bf66e 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -225,7 +225,7 @@ } const rows = materializedViewRows(doc); $('views-meta').innerHTML = stage3.materialized_views_error - ? `no installable plan: ${esc(stage3.materialized_views_error)}` + ? `invalid plan: ${esc(stage3.materialized_views_error)}` : `${rows.length} materialized view${rows.length === 1 ? '' : 's'}` + (rows.length ? '' : ': nothing is materialized across evaluations; every query computes at query time') + (stage3.planner_executor_error diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index 96bf70c48..d238176aa 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -810,7 +810,7 @@ def test_a_document_fills_the_data_workload_and_says_why_the_plan_has_no_views(s self.assertIn("ingesting", js.eval("__els.dataArrival.textContent")) self.assertIn("raw dataretained", self.text(js, "deployment")) self.assertIn("every 10 s · predictable", self.text(js, "queries")) - self.assertIn(f"no installable plan: {doc['stage3_plan_selection']['materialized_views_error']}", self.text(js, "views-meta")) + self.assertIn(f"invalid plan: {doc['stage3_plan_selection']['materialized_views_error']}", self.text(js, "views-meta")) def test_a_plan_the_planners_executor_cannot_run_lists_its_views_and_says_why(self): """A plan the Planner's executor cannot run still lists its views;