diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index e1559b4f1..37c4ff86d 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -14,8 +14,8 @@ asap-stage1-pass2-reuse-candidates = { path = "../stage1-pass2-reuse-candidates" asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-cost = { path = "../cost" } asap-stage3-plan-selection = { path = "../stage3-plan-selection" } -# The installable plan of the selected candidate (`installable_plan_of`), -# whose stored outputs `asap_planner` writes. +# The installable plan of the selected candidate (`InstallablePlan`), +# whose stored outputs and plan file `asap_planner` writes. asap-planner = { path = "../planner" } # The executor's supported summary families, which `asap_planner` plans # with when the deployment declares none, as `plan_installable` does. diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index d5da82ff8..1441df9ef 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -44,13 +44,29 @@ // `query_time_results_retained`, `counter_semantics`, // `state_not_mergeable_across_windows`, `excluded_candidates`; an // omitted field takes its default). -// Without `supported_summaries`, the executor's summary families are -// supported, as `plan_installable` defaults it. +// Without `supported_summaries` (or without the file), the Planner's +// executor's summary families are supported, as `plan_installable` +// defaults it. To plan for a runtime that builds every summary family, +// pass a file listing them all, e.g. +// `tools/dag-viewer/deployment-capabilities/every-summary-family.json`. // - `--ignore-accuracy-guarantees`: Stage 3 accepts every query output // regardless of its accuracy guarantee // (`PlanningModels::ignore_accuracy_guarantees`), comparing candidates // on cost alone. By default it enforces them, as `plan_installable` // does. +// - `--plan-file ` (needs `--run-until stage3`): besides the stage +// output, writes the selected plan as an `InstallablePlan` JSON plan +// file (`InstallablePlan::of_dag`), which a runtime reads with +// `InstallablePlan::from_json`. When Stage 3 selects no plan, no file is +// written (a file already at the path is left as it is) and the run +// exits 1 after writing the stage output. +// +// With a selected plan, the run also compiles it on the Planner's executor +// (`InstallablePlan::check_compiles`) and reports the result: the document's +// `planner_executor_can_run`, and when the executor cannot run the plan a +// line `note: the Planner's executor cannot run this plan: ` on +// standard error. The result never blocks the plan file or the exit status: +// a runtime other than the Planner's executor may run the plan. // // The document (`--format json`): // - workload: the queries (`id`, `language`, `text`, `requirements`, @@ -82,8 +98,11 @@ // `parameters`, `timing`, `lifetime`) and the `queries` reading it. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them; `materialized_views_error` when the -// plan is not installable. `warnings`: a PromQL query applying `rate`, `increase` -// or `irate` to a declared gauge or combining a declared summary's +// plan fails `InstallablePlan::validate` (its views cannot be derived). +// `planner_executor_can_run`: whether the Planner's executor compiles +// the plan, with `planner_executor_error`, its reason, when it does +// not. `warnings`: a PromQL query applying `rate`, `increase` or +// `irate` to a declared gauge or combining a declared summary's // quantile series across series (`metric_type_warnings`), each data fact the selected // plan assumed because the data workload does not declare it (`assumed_data_facts`, // saying whether answers may then be wrong beyond the accuracy target), and a peak @@ -137,7 +156,7 @@ use asap_devtools::planning_stages_text::{ use asap_devtools::stable_output::{rounded_to_12_significant_digits, CanonicalIds}; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_planner::{assumed_data_facts, installable_plan_of, PlanningModels, SelectionError}; +use asap_planner::{assumed_data_facts, InstallablePlan, PlanningModels, SelectionError}; use asap_stage3_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; use asap_types::accuracy::{default_metric_for_root, AccuracyTarget}; use asap_types::deployment_capabilities::DeploymentCapabilities; @@ -160,7 +179,7 @@ use serde_json::{json, Value}; const USAGE: &str = "usage: asap_planner (--workload )... \ --run-until stage0|stage1_pass1|stage1_pass2|stage2|stage3 [--format text|json] [--verbose] [--out ] \ [--sql-catalog-table ]... [--cost-inputs ] [--deployment-capabilities ] \ -[--ignore-accuracy-guarantees]"; +[--ignore-accuracy-guarantees] [--plan-file ]"; fn main() { match run(std::env::args().skip(1).collect()) { @@ -205,6 +224,7 @@ fn run(args: Vec) -> Result<(), Failure> { let mut deployment_capabilities = DeploymentCapabilities::default(); let mut ignore_accuracy_guarantees = false; let mut verbose = false; + let mut plan_file = None; let mut args = args.into_iter(); while let Some(flag) = args.next() { let mut value = || { @@ -234,10 +254,16 @@ fn run(args: Vec) -> Result<(), Failure> { "--printing-full-candidates was renamed to --verbose".into(), )) } + "--plan-file" => plan_file = Some(value()?), other => return Err(Failure::Usage(format!("unknown argument {other}"))), } } let run_until = run_until.ok_or(Failure::Usage("--run-until is required".into()))?; + if plan_file.is_some() && run_until != RunUntil::Stage3 { + return Err(Failure::Usage( + "--plan-file needs --run-until stage3".into(), + )); + } if workload_paths.is_empty() { return Err(Failure::Usage("--workload is required".into())); } @@ -251,7 +277,8 @@ fn run(args: Vec) -> Result<(), Failure> { } let (catalog, table_names) = sql_catalog(&tables)?; let (described, roots, entries, data) = lowered_together(&workloads, &catalog, &table_names)?; - // Plan with the executor's summary families when the deployment declares none. + // Plan with the executor's summary families when the deployment declares + // none. if deployment_capabilities.supported_summaries.is_none() { deployment_capabilities.supported_summaries = Some(asap_executor::supported_summary_families()); @@ -268,14 +295,27 @@ fn run(args: Vec) -> Result<(), Failure> { .and_then(|run| run.run.selection.as_ref().ok()) .and_then(|(plan, _)| asap_cost::plan_cost(plan, &entries, &data, &cost_inputs).ok()) .map(|(cost, _)| cost); - // The selected plan's materialized views, as `plan_installable` derives them. - let materialized_views = result + // The selected plan as a runtime installs it, validated. + let installable = result .as_ref() .ok() .and_then(|run| run.run.selection.as_ref().ok()) - .map(|(plan, _)| materialized_views_of(plan, &entries)); + .map(|(plan, _)| { + let query_texts = entries.iter().map(|entry| entry.query.0.clone()).collect(); + InstallablePlan::of_dag(plan.clone(), query_texts).map_err(|e| e.to_string()) + }); + // Whether the Planner's executor compiles it: reported, never blocking. + let executor_can_run = installable + .as_ref() + .and_then(|installable| installable.as_ref().ok()) + .map(InstallablePlan::check_compiles); + // Its materialized views, as `plan_installable` derives them. + let materialized_views = installable.as_ref().map(|installable| { + let plan = installable.as_ref().map_err(Clone::clone)?; + plan.materialized_views().map_err(|e| e.to_string()) + }); // A stage that failed: the output is still written, then the run exits 1. - let failure = match &result { + let mut failure = match &result { Err(PlanningStagesError::Pass1(error)) => Some(format!("Pass 1: {error}")), Ok(run) => match &run.run.selection { Err(SelectionError::NotRun) | Ok(_) => None, @@ -310,6 +350,12 @@ fn run(args: Vec) -> Result<(), Failure> { )?; // Stage 3 is null when Pass 1 rejected a query. if let Some(stage3) = document["stage3_plan_selection"].as_object_mut() { + if let Some(can_run) = &executor_can_run { + stage3.insert("planner_executor_can_run".into(), json!(can_run.is_ok())); + if let Err(reason) = can_run { + stage3.insert("planner_executor_error".into(), json!(reason)); + } + } if !warnings.is_empty() { stage3.insert("warnings".into(), json!(warnings)); } @@ -355,6 +401,23 @@ fn run(args: Vec) -> Result<(), Failure> { Some(path) => std::fs::write(&path, text).map_err(|e| format!("{path}: {e}"))?, None => print!("{text}"), } + if let Some(Err(reason)) = &executor_can_run { + eprintln!("note: the Planner's executor cannot run this plan: {reason}"); + } + if let Some(path) = plan_file { + match installable { + Some(Ok(plan)) => { + let json = serde_json::to_string_pretty(&plan).map_err(|e| e.to_string())?; + std::fs::write(&path, json + "\n") + .map_err(|e| format!("--plan-file {path}: {e}"))?; + } + Some(Err(error)) => failure = Some(format!("--plan-file {path}: {error}")), + // No plan was selected: `failure` already says why. + None => { + failure = failure.map(|error| format!("{error} (no plan file written to {path})")) + } + } + } match failure { Some(error) => Err(Failure::Other(error)), None => Ok(()), @@ -702,18 +765,6 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option Result, String> { - let plan = installable_plan_of(plan.clone(), entries).map_err(|e| e.to_string())?; - plan.materialized_views().map_err(|e| e.to_string()) -} - /// Stage 2's physical candidates $\mathcal C$: each with its logical /// candidate (`group`, `index_in_group`), its `properties` ($\tau$, /// $\ell$) and, per input $(c, i)$, the input `group` and, as `edges`, diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index db141c0b6..482bf6981 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -2377,3 +2377,302 @@ fn the_default_text_is_the_terse_form_and_verbose_adds_the_details() { } } } + +// ── Plan file (`--plan-file`) and the Planner's executor check ────────────── + +/// What a plan-file path holds before a run: a file an earlier run left. +const EARLIER_PLAN_FILE: &str = "a plan file an earlier run left"; + +/// A plan-file path holding [`EARLIER_PLAN_FILE`]: a run that selects a +/// plan overwrites it; any other run leaves it as it is. +fn earlier_plan_file() -> TempFile { + TempFile::new(EARLIER_PLAN_FILE) +} + +/// The plan file at `path`, read back as a runtime reads it and validated. +fn read_plan_file(path: &str) -> asap_planner::InstallablePlan { + let text = std::fs::read_to_string(path).unwrap(); + let plan = asap_planner::InstallablePlan::from_json(&text).unwrap(); + plan.validate().unwrap(); + assert_eq!( + plan.wire_version, + asap_planner::INSTALLABLE_PLAN_WIRE_VERSION + ); + plan +} + +/// Whether `plan`'s DAG is the plan `document` selected; the document's +/// numbers are rounded to 12 significant digits, so the file's are too. +fn is_the_selected_plan(plan: &asap_planner::InstallablePlan, document: &Value) -> bool { + asap_devtools::stable_output::rounded_to_12_significant_digits( + serde_json::to_value(&plan.dag).unwrap(), + ) == document["stage3_plan_selection"]["plan"] +} + +/// The note `asap_planner` writes on standard error when the Planner's +/// executor cannot run the selected plan. +const EXECUTOR_CANNOT_RUN_NOTE: &str = "note: the Planner's executor cannot run this plan: "; + +/// The ready-to-use capabilities file that supports every summary family +/// with every readout it answers. +fn every_summary_family_file() -> String { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tools/dag-viewer/deployment-capabilities/every-summary-family.json") + .to_str() + .unwrap() + .to_owned() +} + +/// `--plan-file` writes the selected plan of the committed `example1` +/// workload as `InstallablePlan` JSON: the file reads back with +/// `InstallablePlan::from_json`, validates, holds the workload's query +/// texts in order, and its DAG is the plan the document selected. The +/// Planner's executor runs it: the document says so and there is no note. +#[test] +fn plan_file_round_trips_the_selected_plan() { + let workload: PlanningWorkload = + serde_json::from_str(&std::fs::read_to_string(example("example1")).unwrap()).unwrap(); + let query_texts: Vec = workload + .query_workload + .entries() + .map(|entry| entry.query.0.clone()) + .collect(); + let file = earlier_plan_file(); + let path = example("example1"); + let args = [ + "--workload", + path.as_str(), + "--run-until", + "stage3", + "--plan-file", + file.path(), + "--format", + "json", + ]; + let output = planner(&args); + assert_eq!(output.status.code(), Some(0)); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(!stderr.contains(EXECUTOR_CANNOT_RUN_NOTE), "{stderr}"); + let document: Value = serde_json::from_slice(&output.stdout).unwrap(); + let plan = read_plan_file(file.path()); + assert_eq!(plan.query_texts, query_texts); + assert!(is_the_selected_plan(&plan, &document)); + let selection = &document["stage3_plan_selection"]; + assert_eq!(selection["planner_executor_can_run"], json!(true)); + assert!(selection["planner_executor_error"].is_null(), "{selection}"); +} + +/// `absent_over_time(data[1m])`: Stages 1-3 plan it, but the Planner's +/// executor has no `absent_over_time` kernel. The plan file is still +/// written and the run exits 0; the executor's gap is only reported: in +/// the document (`planner_executor_can_run: false` with its reason) and in +/// one note on standard error. The document lists the plan's views. +#[test] +fn a_plan_the_executor_cannot_run_is_written_with_a_note() { + let query = "absent_over_time(data[1m])"; + let workload = TempFile::json(&adhoc(&[query], exact(), &["data"])); + let file = earlier_plan_file(); + let output = planner(&[ + "--workload", + workload.path(), + "--run-until", + "stage3", + "--plan-file", + file.path(), + "--format", + "json", + ]); + assert_eq!(output.status.code(), Some(0)); + let stderr = String::from_utf8(output.stderr).unwrap(); + let notes: Vec<&str> = stderr + .lines() + .filter(|line| line.starts_with(EXECUTOR_CANNOT_RUN_NOTE)) + .collect(); + assert_eq!(notes.len(), 1, "{stderr}"); + let document: Value = serde_json::from_slice(&output.stdout).unwrap(); + let selection = &document["stage3_plan_selection"]; + // Precondition: an executor-only gap. Once the executor runs + // `absent_over_time`, pick another query it plans but cannot compile. + assert_eq!(selection["planner_executor_can_run"], json!(false)); + let reason = selection["planner_executor_error"].as_str().unwrap(); + assert!(reason.contains("AbsentOverTime"), "{reason}"); + assert_eq!(notes[0], format!("{EXECUTOR_CANNOT_RUN_NOTE}{reason}")); + assert!(selection["materialized_views"].is_array(), "{selection}"); + assert!( + selection["materialized_views_error"].is_null(), + "{selection}" + ); + let plan = read_plan_file(file.path()); + assert_eq!(plan.query_texts, [query]); + assert!(is_the_selected_plan(&plan, &document)); + assert_eq!(plan.check_compiles().unwrap_err(), reason); +} + +/// When Stage 3 selects no plan (example 3b under cost model 2, which plan +/// selection does not implement), the run exits 1, writes no plan file, +/// says so, and leaves the file already at the path as it is. +#[test] +fn no_selected_plan_exits_1_and_leaves_the_plan_file_path_alone() { + let cost = cost_inputs(&asap_cost::CostInputs { + cost_model: asap_cost::CostModel::ProvisionedCapacity( + asap_cost::ResourceUnitCost::illustrative(), + ), + ..Default::default() + }); + let file = earlier_plan_file(); + let path = example("example3b"); + let output = planner(&[ + "--workload", + path.as_str(), + "--run-until", + "stage3", + "--cost-inputs", + cost.path(), + "--plan-file", + file.path(), + "--format", + "json", + ]); + assert_eq!(output.status.code(), Some(1)); + let document: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!(!selects_a_plan(&document["stage3_plan_selection"])); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains(&format!("no plan file written to {}", file.path())), + "{stderr}" + ); + assert_eq!( + std::fs::read_to_string(file.path()).unwrap(), + EARLIER_PLAN_FILE + ); +} + +/// A run that fails before planning (here an unreadable workload file) +/// leaves the file already at `--plan-file` as it is. +#[test] +fn a_failed_run_leaves_the_plan_file_path_alone() { + let file = earlier_plan_file(); + refused(&[ + "--workload", + "/nonexistent/workload.json", + "--run-until", + "stage3", + "--plan-file", + file.path(), + ]); + assert_eq!( + std::fs::read_to_string(file.path()).unwrap(), + EARLIER_PLAN_FILE + ); +} + +/// The summary families `asap_planner` plans with (the document shows the +/// capabilities it hands to planning) come from `--deployment-capabilities` +/// alone. Without the file, or with a file that does not list +/// `supported_summaries`, they are the Planner's executor's families, as +/// `plan_installable` defaults them; a file that lists them is used as it +/// is: here the ready-to-use file with every summary family, and a file +/// with KLL only. +#[test] +fn summary_families_come_from_the_deployment_capabilities() { + let workload = adhoc(&["count by (job) (m)"], within(0.01, 0.01), &["m"]); + let planned_with = |args: &[&str]| { + let mut all = vec!["--run-until", "stage0"]; + all.extend_from_slice(args); + document(&workload, &all)["deployment_capabilities"]["supported_summaries"].clone() + }; + let executor_families = + serde_json::to_value(asap_executor::supported_summary_families()).unwrap(); + assert_eq!(planned_with(&[]), executor_families); + let without_summaries = TempFile::json(&json!({"raw_data_retained": true})); + assert_eq!( + planned_with(&["--deployment-capabilities", without_summaries.path()]), + executor_families + ); + let every_family = every_summary_family_file(); + let declared: Value = + serde_json::from_str(&std::fs::read_to_string(&every_family).unwrap()).unwrap(); + assert_eq!( + planned_with(&["--deployment-capabilities", &every_family]), + declared["supported_summaries"] + ); + let kll = json!([{"summary_type": "Kll", "readouts": ["Quantile"]}]); + let kll_only = TempFile::json(&json!({ "supported_summaries": kll })); + assert_eq!( + planned_with(&["--deployment-capabilities", kll_only.path()]), + kll + ); +} + +/// `tools/dag-viewer/deployment-capabilities/every-summary-family.json` +/// lists every sketch algorithm with every readout its capability table +/// row answers, so its `supported_summaries` rule out no summary type and +/// no readout; its other fields keep their defaults. +#[test] +fn the_every_summary_family_file_rules_out_nothing() { + use asap_types::deployment_capabilities::{DeploymentCapabilities, ExclusionSource}; + use asap_types::summary_type::{SketchAlgorithm, SupportedSummary}; + let capabilities: DeploymentCapabilities = + serde_json::from_str(&std::fs::read_to_string(every_summary_family_file()).unwrap()) + .unwrap(); + let every_family: Vec = SketchAlgorithm::ALL + .iter() + .map(|&algorithm| SupportedSummary { + summary_type: algorithm, + readouts: algorithm + .capability() + .readouts + .iter() + .map(|&(statistic, _)| statistic) + .collect(), + }) + .collect(); + assert_eq!( + capabilities.supported_summaries.as_ref(), + Some(&every_family), + "expected:\n{}", + serde_json::to_string_pretty(&every_family).unwrap() + ); + let summary_exclusions: Vec<_> = capabilities + .exclusions() + .into_iter() + .filter(|exclusion| { + matches!( + exclusion.source, + ExclusionSource::SummaryNotSupported | ExclusionSource::ReadoutNotSupported + ) + }) + .collect(); + assert!(summary_exclusions.is_empty(), "{summary_exclusions:?}"); +} + +/// `--plan-file` without its path and `--plan-file` before Stage 3 selects +/// a plan are usage errors (status 2, with the usage line) that leave the +/// plan-file path alone. +#[test] +fn plan_file_flag_misuse_is_a_usage_error() { + let workload = TempFile::json(&adhoc(&["sum(data)"], exact(), &["data"])); + let file = earlier_plan_file(); + for (args, message) in [ + ( + vec!["--run-until", "stage3", "--plan-file"], + "--plan-file needs a value", + ), + ( + vec!["--run-until", "stage2", "--plan-file", file.path()], + "--plan-file needs --run-until stage3", + ), + ] { + let mut all = vec!["--workload", workload.path()]; + all.extend(args); + let output = planner(&all); + assert_eq!(output.status.code(), Some(2), "{all:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains(message), "{stderr}"); + assert!(stderr.contains("usage: asap_planner"), "{stderr}"); + } + assert_eq!( + std::fs::read_to_string(file.path()).unwrap(), + EARLIER_PLAN_FILE + ); +} diff --git a/crates/devtools/tests/corpus_snapshots.rs b/crates/devtools/tests/corpus_snapshots.rs index 8f163b991..b340ab8e8 100644 --- a/crates/devtools/tests/corpus_snapshots.rs +++ b/crates/devtools/tests/corpus_snapshots.rs @@ -122,7 +122,12 @@ fn plan_snapshot(document: &Value) -> Map { "accepted": stage3["accepted_accuracy_per_query"], }), ); - for key in ["materialized_views", "materialized_views_error", "warnings"] { + for key in [ + "materialized_views", + "materialized_views_error", + "planner_executor_error", + "warnings", + ] { if let Some(value) = stage3.get(key) { snapshot.insert(key.into(), value.clone()); } diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json index 621527658..a0535ab49 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json @@ -1,7 +1,78 @@ { "accuracy_guarantees": "enforced", - "materialized_views": [], - "materialized_views_error": "the executor cannot compile the plan's query run: invalid DAG: node 13: invalid DAG: value operation has no native implementation", + "materialized_views": [ + { + "coverage": { + "definition_id": "7e48d8927e84a32d499eaaf08711a04d", + "selection": [ + { + "columns": [ + { + "column": { + "column": "job", + "scan_ordinal": 0, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "constraint": { + "in": [ + { + "Utf8": "order-service" + } + ] + } + } + ] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 1, + "refresh": "at_ingestion", + "retention_ms": 3900000 + }, + { + "coverage": { + "definition_id": "58395555aae268f4f68a8a3e82a6c45b", + "selection": [ + { + "columns": [ + { + "column": { + "column": "job", + "scan_ordinal": 0, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "constraint": { + "in": [ + { + "Utf8": "order-service" + } + ] + } + } + ] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 7, + "refresh": "at_ingestion", + "retention_ms": 3900000 + } + ], + "planner_executor_error": "the executor cannot compile the plan's query run: invalid DAG: node 13: invalid DAG: value operation has no native implementation", "query": "max_over_time((sum(rate(http_requests_total{job=\"order-service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"order-service\"}[5m])))[6h:1m])", "selected": { "accepted": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json index 1f26986f6..af918f144 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json @@ -1,7 +1,43 @@ { "accuracy_guarantees": "enforced", - "materialized_views": [], - "materialized_views_error": "the executor cannot compile the plan's query run: invalid DAG: node 7: invalid DAG: value operation has no native implementation", + "materialized_views": [ + { + "coverage": { + "definition_id": "be9c29c7ba723e7271224b1879a6c7b4", + "selection": [ + { + "columns": [ + { + "column": { + "column": "job", + "scan_ordinal": 0, + "source": { + "TimeSeries": { + "metric": "http_request_duration_seconds_bucket" + } + } + }, + "constraint": { + "in": [ + { + "Utf8": "user-service" + } + ] + } + } + ] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 1, + "refresh": "at_ingestion", + "retention_ms": 3900000 + } + ], + "planner_executor_error": "the executor cannot compile the plan's query run: invalid DAG: node 7: invalid DAG: value operation has no native implementation", "query": "max_over_time(histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"user-service\"}[5m])) by (le))[12h:1m])", "selected": { "accepted": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json index 779c7b5d5..dca914613 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json @@ -1,7 +1,7 @@ { "accuracy_guarantees": "enforced", "materialized_views": [], - "materialized_views_error": "the executor cannot compile the plan's query run: invalid DAG: node 3: invalid DAG: value operation has no native implementation", + "planner_executor_error": "the executor cannot compile the plan's query run: invalid DAG: node 3: invalid DAG: value operation has no native implementation", "query": "max_over_time(sum(process_resident_memory_bytes)[6h:1m])", "selected": { "accepted": [ diff --git a/crates/planner/src/installable/mod.rs b/crates/planner/src/installable/mod.rs index 166f9ecad..5290f93ad 100644 --- a/crates/planner/src/installable/mod.rs +++ b/crates/planner/src/installable/mod.rs @@ -234,15 +234,18 @@ pub async fn plan_installable( /// The installable plan of `plan`, the plan Stage 3 built for a workload /// whose entries are `entries` (in `QueryWorkload::entries()` order): what -/// [`plan_installable`] returns after planning, without its cost report. A -/// caller that runs the planning stages itself (e.g. a developer tool) uses -/// it to see what a runtime would install. +/// [`plan_installable`] returns after planning, without its cost report. +/// [`InstallablePlan::of_dag`], with each view's refresh and the query run +/// compiled ([`executor_compiles`]): a plan the Planner's executor cannot +/// run is refused here, not installed. pub fn installable_plan_of( plan: PhysicalASAPDAG, entries: &[QueryWorkloadEntry], ) -> Result { let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - InstallablePlan::of_dag(plan, query_texts) + let plan = InstallablePlan::of_dag(plan, query_texts)?; + executor_compiles(&plan.dag, &plan.materialized_views()?)?; + Ok(plan) } impl InstallablePlan { @@ -296,7 +299,8 @@ impl InstallablePlan { /// DAG's own `TimeRange`s are compiled, so a window the executor cannot /// run fails here. `Err` holds the executor's error, or why the views /// cannot be derived. [`plan_installable`] runs the same compiles before - /// it returns a plan; this checks a plan read from a plan file. + /// it returns a plan; this checks a plan read from a plan file or built + /// by [`Self::of_dag`]. pub fn check_compiles(&self) -> Result<(), String> { let views = self .materialized_views() @@ -304,25 +308,24 @@ impl InstallablePlan { executor_compiles(&self.dag, &views).map_err(|error| error.to_string()) } - /// The plan of `dag`, with each view's and query-time scan's retention - /// (which Stage 3 set) checked ([`PhysicalASAPDAG::materialized_views`], - /// [`PhysicalASAPDAG::raw_data_retention`]), and each view's refresh - /// and the query run compiled ([`executor_compiles`]): a plan the - /// executor cannot run is refused here, not installed. - pub(crate) fn of_dag( + /// The plan of `dag` answering `query_texts` (one per root, in workload + /// order), checked as [`Self::validate`] checks a plan file: the DAG's + /// validity, one text per root, no standalone scalar query, and each + /// view's and query-time scan's retention, which Stage 3 set. Not + /// compiled by the Planner's executor ([`Self::check_compiles`] does + /// that), so a runtime other than the Planner's executor gets every plan + /// Stage 3 selects: e.g. a plan with an `absent_over_time`, which the + /// Planner's executor has no kernel for. + pub fn of_dag( dag: PhysicalASAPDAG, query_texts: Vec, ) -> Result { - let views = dag.materialized_views()?; - dag.raw_data_retention()?; - check_queries(&dag, &query_texts)?; - executor_compiles(&dag, &views)?; let plan = InstallablePlan { wire_version: INSTALLABLE_PLAN_WIRE_VERSION, dag, query_texts, }; - debug_assert!(plan.validate().is_ok()); + plan.validate()?; Ok(plan) } diff --git a/crates/planner/src/installable/tests.rs b/crates/planner/src/installable/tests.rs index 1386370a5..b58129889 100644 --- a/crates/planner/src/installable/tests.rs +++ b/crates/planner/src/installable/tests.rs @@ -4,7 +4,7 @@ use std::rc::Rc; thread_local! { /// The plan nodes handed to the executor's compiler by the views' - /// refreshes in [`InstallablePlan::of_dag`] and + /// refreshes in [`installable_plan_of`] and /// [`InstallablePlan::check_compiles`], summed over their compiles. pub(super) static COMPILED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; } @@ -49,6 +49,7 @@ async fn view_refreshes_compile_only_the_nodes_below_them() { COMPILED_NODES.with(|count| count.set(0)); let rebuilt = InstallablePlan::of_dag(plan.dag.clone(), plan.query_texts.clone()).unwrap(); assert_eq!(rebuilt.materialized_views().unwrap().len(), views); + rebuilt.check_compiles().unwrap(); plan.check_compiles().unwrap(); let compiled = COMPILED_NODES.with(std::cell::Cell::get); assert!( @@ -372,3 +373,52 @@ fn a_root_time_shift_is_read_whole_by_the_installer_not_by_the_cost() { assert_eq!(retention.len(), 1); assert_eq!(retention[0].lookback_ms(), None); } + +/// `absent_over_time(data[1m])` every 10 s, exact: Stages 1-3 +/// plan it, but the Planner's executor has no `absent_over_time` kernel. +/// [`InstallablePlan::of_dag`] still returns the plan, validated, for any +/// runtime; [`InstallablePlan::check_compiles`] reports the executor's gap, +/// and [`installable_plan_of`] (what `plan_installable` returns) refuses it. +#[tokio::test] +async fn of_dag_validates_a_plan_the_executor_cannot_compile_without_refusing_it() { + let workload: crate::PlanningWorkload = serde_json::from_str( + r#"{"query_workload": {"language": "promql", "repeating_queries": [{ + "query": "absent_over_time(data[1m])", + "demand": {"fixed_interval_at": {"interval": 10000, "evaluation_phase": 0}}, + "requirements": {"accuracy": {"max_error": 0.0, "failure_probability": 0.0}}, + "predictability": {"predictable": {"known_at": null}}}]}, + "data_workload": {"arrival": {"ingesting": {"rows_per_second": 10000.0}}, + "sources": {"data": {"series_count": 1000, "metric_type": "gauge", + "sample_interval": 100}}}}"#, + ) + .unwrap(); + let output = crate::e2e_plan(crate::UserInput::new( + &workload, + crate::FrontendInput::Promql, + crate::PlanningModels::builtin(), + )) + .await + .expect("Stages 1-3 plan absent_over_time"); + let query_texts = vec!["absent_over_time(data[1m])".to_owned()]; + let plan = InstallablePlan::of_dag(output.plan.clone(), query_texts) + .expect("validated without the executor's compile check"); + plan.validate().unwrap(); + // Precondition: an executor-only gap. Once the executor runs + // `absent_over_time`, pick another query it plans but cannot compile. + assert!(plan.check_compiles().is_err()); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let refused = installable_plan_of(output.plan.clone(), &entries).unwrap_err(); + assert!( + matches!( + refused, + InstallablePlanError::QueryRunNotExecutable { .. } + | InstallablePlanError::MaterializedViewNotExecutable { .. } + ), + "{refused}" + ); + // Still validated: one query text per root. + assert!(matches!( + InstallablePlan::of_dag(output.plan, Vec::new()), + Err(InstallablePlanError::QueriesDoNotMatchRoots { roots: 1 }) + )); +} diff --git a/docs/user_guide_docs/asap-planner.md b/docs/user_guide_docs/asap-planner.md index 3ccd5bbb9..fd6cc7f98 100644 --- a/docs/user_guide_docs/asap-planner.md +++ b/docs/user_guide_docs/asap-planner.md @@ -36,7 +36,7 @@ minutes in a debug build. asap_planner (--workload )... --run-until stage0|stage1_pass1|stage1_pass2|stage2|stage3 [--format text|json] [--verbose] [--out ] [--sql-catalog-table ]... [--cost-inputs ] [--deployment-capabilities ] - [--ignore-accuracy-guarantees] + [--ignore-accuracy-guarantees] [--plan-file ] ``` | Option | Meaning | @@ -50,6 +50,7 @@ asap_planner (--workload )... --run-until stage0|stage1_pass1|stage1_pass2 | `--cost-inputs ` | The cost model and its costs (`CostInputs` JSON). The files in `tools/dag-viewer/cost-inputs/` are ready to use. Without it, the built-in usage-over-time model is used. Stage 3 costs only usage-over-time models (`aws-fargate-us-east-1.json` is one); a provisioned- or cluster-capacity model ends Stage 3 with an `UnsupportedCostModel` error. | | `--deployment-capabilities ` | What the deployment can do, e.g. `{"raw_data_retained": true, "query_time_results_retained": false}`. Set it to match your deployment; see [Deployment capabilities](#deployment-capabilities). | | `--ignore-accuracy-guarantees` | Accept candidates without an accuracy guarantee. By default each query's accuracy requirement is enforced. | +| `--plan-file ` | Also write the plan Stage 3 selects to this file, as `InstallablePlan` JSON; needs `--run-until stage3`. See [Write a plan file](#write-a-plan-file). | Any other option, or a missing required one, is a usage error (exit status 2). An input the planner cannot read exits with status 1. So does a @@ -57,10 +58,33 @@ run in which a stage failed (for example, Pass 1 rejected a query, or no plan meets the requirements): the output is still written, with the error under the stage that failed. +With a selected plan, `asap_planner` also compiles it on the Planner's +executor and reports the result; it never changes the exit status. When the +executor cannot run the plan, standard error gets one line + +```text +note: the Planner's executor cannot run this plan: +``` + +and the JSON document's `stage3_plan_selection` has +`"planner_executor_can_run": false` with the reason in +`planner_executor_error` (otherwise `"planner_executor_can_run": true`). + ## Deployment capabilities The `--deployment-capabilities` file is a `DeploymentCapabilities` JSON -object; an omitted field keeps its default. Besides +object; an omitted field keeps its default. + +`supported_summaries` lists the sketch summary types the deployment builds, +each with the statistics it reads from them, e.g. +`[{"summary_type": "Kll", "readouts": ["Quantile"]}]`. Without the file, or +when the file leaves it out, the planner uses the summary types the +Planner's executor builds. To plan for a runtime that builds every summary +type, pass the ready-to-use +[`every-summary-family.json`](../../tools/dag-viewer/deployment-capabilities/every-summary-family.json), +which lists each one with every statistic it answers. + +Besides `supported_summaries`, `raw_data_retained` and `query_time_results_retained`, three fields say what the executor cannot do or what the caller rules out: @@ -141,6 +165,29 @@ cargo run --release -p asap-devtools --bin asap_planner -- \ See [Run and inspect a query](run-a-query.md) for workload, source-fact and SQL examples. To try a single query, write a one-query workload file. +## Write a plan file + +`--plan-file ` writes the selected plan as an `InstallablePlan` JSON +file (`wire_version`, the selected DAG and the query texts), the input a +runtime installs; a runtime reads it with `InstallablePlan::from_json`. The +plan file is written whether or not the Planner's executor can run the plan: +another runtime may run it. + +```sh +cargo run --release -p asap-devtools --bin asap_planner -- \ + --workload tools/dag-viewer/workloads/planner-layering/example1.json \ + --run-until stage3 --plan-file plan.json +``` + +| Run | Exit status | `plan.json` | +| --- | --- | --- | +| Stage 3 selects a plan the Planner's executor runs | 0 | written | +| Stage 3 selects a plan the Planner's executor cannot run | 0, with the `note:` line on standard error | written | +| Stage 3 selects no plan, or the run fails | 1, with `asap_planner: (no plan file written to plan.json)` when Stage 3 selected none | not written; a file already there is left as it is | + +`--plan-file` with a `--run-until` before `stage3` is a usage error (exit +status 2). + ## Inspect the result With `--format json`, open the document in the diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index e2e507fa6..4ed08f55a 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -151,7 +151,10 @@ whole document): `pruned_candidates` (as Stage 2's), `milp_variables`, `milp_constraints`, `materialized_views` (`node_id`, a plan node, `refresh`, `coverage.definition_id`, `inputs` and `retention_ms`), - `materialized_views_error`, `warnings` and `error`; + `materialized_views_error` (the plan is invalid: its views cannot be + derived), `planner_executor_can_run` (whether the + Planner's executor compiles the plan) with `planner_executor_error` when + it does not, `warnings` and `error`; - `memo.steps[].groups[]`: `id`, `label`, `operator` (the group's expression as `Operator` JSON, children as group ids), `children`, `parents`, `queries`, `roots_of`, `candidate_count`, diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index 4f6a213d9..54e4bf66e 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -225,9 +225,12 @@ } const rows = materializedViewRows(doc); $('views-meta').innerHTML = stage3.materialized_views_error - ? `no installable plan: ${esc(stage3.materialized_views_error)}` + ? `invalid plan: ${esc(stage3.materialized_views_error)}` : `${rows.length} materialized view${rows.length === 1 ? '' : 's'}` + - (rows.length ? '' : ': nothing is materialized across evaluations; every query computes at query time'); + (rows.length ? '' : ': nothing is materialized across evaluations; every query computes at query time') + + (stage3.planner_executor_error + ? ` · the Planner's executor cannot run this plan: ${esc(stage3.planner_executor_error)}` + : ''); $('materializedViews').innerHTML = rows.length ? table( ['plan node', 'definition', 'kind', 'refresh', 'reads', 'retention', 'Stage 3 cost', 'read by'], rows.map((r) => `${esc(r.nodeId)}` + diff --git a/tools/dag-viewer/deployment-capabilities/every-summary-family.json b/tools/dag-viewer/deployment-capabilities/every-summary-family.json new file mode 100644 index 000000000..cb8d73a6b --- /dev/null +++ b/tools/dag-viewer/deployment-capabilities/every-summary-family.json @@ -0,0 +1,32 @@ +{ + "supported_summaries": [ + {"summary_type": "Cms", "readouts": ["ItemFrequency", "Total"]}, + {"summary_type": "CountSketch", "readouts": ["ItemFrequency"]}, + {"summary_type": "CountSketchWithL2Norm", "readouts": ["ItemFrequency", "FrequencyL2"]}, + {"summary_type": "CmsWithHeap", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CountSketchWithHeap", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CmsWithSpaceSaving", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CountSketchWithSpaceSaving", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CmsFold", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "CountSketchFold", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "Hll", "readouts": ["Distinct"]}, + {"summary_type": "HllHip", "readouts": ["Distinct"]}, + {"summary_type": "Kmv", "readouts": ["Distinct"]}, + {"summary_type": "Kll", "readouts": ["Quantile", "Rank"]}, + {"summary_type": "KllDynamic", "readouts": ["Quantile", "Rank"]}, + {"summary_type": "DDSketch", "readouts": ["Quantile"]}, + {"summary_type": "SpaceSaving", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "Elastic", "readouts": ["ItemFrequency", "TopK"]}, + {"summary_type": "Coco", "readouts": ["ItemFrequency", "PartialKeySum"]}, + {"summary_type": "Bloom", "readouts": ["Membership"]}, + {"summary_type": "SetAggregator", "readouts": ["Distinct", "Membership"]}, + {"summary_type": "UnivMon", "readouts": ["Total", "FrequencyL2", "Distinct", "FrequencyEntropy", "GSum"]}, + {"summary_type": "UnivMonTwoTier", "readouts": ["Total", "FrequencyL2", "Distinct", "FrequencyEntropy", "GSum"]}, + {"summary_type": "UnivMonQ", "readouts": ["ItemFrequency", "FrequencyL2", "Distinct", "Quantile", "Rank", "GSum", "FrequencyEntropy", "TopK"]}, + {"summary_type": "HydraCms", "readouts": ["ItemFrequency", "Total"]}, + {"summary_type": "HydraCountSketch", "readouts": ["ItemFrequency"]}, + {"summary_type": "HydraKll", "readouts": ["Quantile", "Rank"]}, + {"summary_type": "HydraHll", "readouts": ["Distinct"]}, + {"summary_type": "HydraUnivMon", "readouts": ["Total", "Distinct", "FrequencyL2", "FrequencyEntropy"]} + ] +} diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index f60d85c20..d238176aa 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -801,16 +801,29 @@ def test_a_document_fills_the_data_workload_and_says_why_the_plan_has_no_views(s """A planned workload fills the data workload and deployment panels; a plan that is not installable says why under materialized views.""" doc = example1() - # Example 1's plan installs; a plan the executor cannot compile has + # Example 1's plan installs; a plan whose views cannot be derived has # this error instead of its views. doc["stage3_plan_selection"].pop("materialized_views") - doc["stage3_plan_selection"]["materialized_views_error"] = "the executor cannot compile the plan's query run" + doc["stage3_plan_selection"]["materialized_views_error"] = "materialized view 3 reads a view refreshed later" js = self.page("?doc=s.json", {"s.json": json.dumps(doc)}) self.assertIn("q1, q2", self.text(js, "dataSources")) self.assertIn("ingesting", js.eval("__els.dataArrival.textContent")) self.assertIn("raw dataretained", self.text(js, "deployment")) self.assertIn("every 10 s · predictable", self.text(js, "queries")) - self.assertIn(f"no installable plan: {doc['stage3_plan_selection']['materialized_views_error']}", self.text(js, "views-meta")) + self.assertIn(f"invalid plan: {doc['stage3_plan_selection']['materialized_views_error']}", self.text(js, "views-meta")) + + def test_a_plan_the_planners_executor_cannot_run_lists_its_views_and_says_why(self): + """A plan the Planner's executor cannot run still lists its views; + the views line says why the executor cannot run it.""" + doc = example1() + doc["stage3_plan_selection"].pop("materialized_views_error", None) + doc["stage3_plan_selection"]["materialized_views"] = [materialized_view(2)] + doc["stage3_plan_selection"]["planner_executor_can_run"] = False + doc["stage3_plan_selection"]["planner_executor_error"] = "AbsentOverTime has no native implementation" + js = self.page("?doc=s.json", {"s.json": json.dumps(doc)}) + self.assertIn("1 materialized view", self.text(js, "views-meta")) + self.assertIn("the Planner's executor cannot run this plan: AbsentOverTime has no native implementation", + self.text(js, "views-meta")) def test_materialized_views_table_has_one_row_per_view_and_a_row_selects_its_node(self): """The materialized views table has one row per view; clicking it diff --git a/tools/validate_joint_workloads.py b/tools/validate_joint_workloads.py index 7c9b227e7..dc178a232 100644 --- a/tools/validate_joint_workloads.py +++ b/tools/validate_joint_workloads.py @@ -80,6 +80,7 @@ def run(entry): result['selected'] = selected result['error'] = selection.get('error') or (document.get('stage1_logical_asap_optimization') or {}).get('error') result['materialized_views_error'] = selection.get('materialized_views_error') + result['planner_executor_error'] = selection.get('planner_executor_error') report = selection.get('search_report') or {} result['search'] = report if selected: @@ -110,6 +111,7 @@ def run(entry): and result.get('guarantees_enforced') and result['cost_certificate_valid'] and result.get('accepted_queries') == result['expected_queries'] and not result.get('error') and not result.get('materialized_views_error') + and not result.get('planner_executor_error') ) print(json.dumps({k: result.get(k) for k in ['workload', 'wall_seconds', 'selected', 'accepted_queries', 'modeled_cost_per_second', 'error']}), flush=True) return result