Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 49 additions & 41 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 5 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ members = [
"crates/types",
"crates/frontend-common",
"crates/sql-function-catalog",
"crates/logical-asap-optimization",
"crates/physical-asap-optimization",
"crates/stage0-memo",
"crates/stage1-pass1-per-computation-candidates",
"crates/stage1-pass2-reuse-candidates",
"crates/stage2-physical-optimization",
"crates/accuracy",
"crates/cost",
"crates/plan-selection",
"crates/stage3-plan-selection",
"crates/frontend-promql",
"crates/frontend-metricsql",
"crates/metricsql-common-parser-support",
Expand Down
5 changes: 0 additions & 5 deletions crates/cost/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,3 @@ serde_json = "1"
# The executor's charge for each summary state, checked against the bytes
# the state byte models give for it plus the accumulator's own fields (#947).
asap-executor = { path = "../executor" }
# Physical plans of PromQL queries for the plan cost tests (tests/), built
# by the stages above this crate; `tests/held_memory.rs` builds them with
# the reference pricer's `optimize_physical_asap`.
asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] }
asap-physical-asap-optimization = { path = "../physical-asap-optimization" }
2 changes: 1 addition & 1 deletion crates/cost/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//! [`NodeUsage`]; [`plan_cost`] the whole plan's [`PlanCost`]. Stage 3
//! computes no cost of its own.
//!
//! Plan selection (Stage 3, `asap-plan-selection`) documents how it prices
//! Plan selection (Stage 3, `asap-stage3-plan-selection`) documents how it prices
//! a candidate with these.
mod measured;
mod model;
Expand Down
2 changes: 1 addition & 1 deletion crates/cost/src/plan/cost.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Costing a physical plan DAG node by node: each node's resource usage
//! ([`node_usage`]), then the plan's usage, latency, retention, retained
//! tumbling window chains and cost ([`plan_cost`]). Moved from Stage 3
//! (`asap-plan-selection`, whose module doc, "Cost model", describes the
//! (`asap-stage3-plan-selection`, whose module doc, "Cost model", describes the
//! model), so Stage 3 computes no cost of its own.

use std::collections::{BTreeMap, HashMap, HashSet};
Expand Down
12 changes: 7 additions & 5 deletions crates/devtools/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@ edition = "2021"
asap-frontend-promql = { path = "../frontend-promql" }
asap-frontend-sql = { path = "../frontend-sql" }
asap-accuracy = { path = "../accuracy" }
asap-logical-asap-optimization = { path = "../logical-asap-optimization" }
asap-physical-asap-optimization = { path = "../physical-asap-optimization" }
asap-stage0-memo = { path = "../stage0-memo" }
asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates" }
asap-stage1-pass2-reuse-candidates = { path = "../stage1-pass2-reuse-candidates" }
asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" }
asap-cost = { path = "../cost" }
asap-plan-selection = { path = "../plan-selection" }
asap-stage3-plan-selection = { path = "../stage3-plan-selection" }
# The installable plan of the selected candidate (`installable_plan_of`),
# whose stored outputs `asap_planner` writes.
asap-planner = { path = "../planner" }
Expand All @@ -34,9 +36,9 @@ tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] }

[dev-dependencies]
asap-types = { path = "../types", features = ["test-support"] }
asap-physical-asap-optimization = { path = "../physical-asap-optimization" }
asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" }
# Pass 1's target list (`per_computation_candidates`) for the tests.
asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] }
asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] }


[[test]]
Expand Down
14 changes: 3 additions & 11 deletions crates/devtools/src/bin/asap_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,8 @@
// - memo: `nodes`, one node table for the section (each distinct node
// once, children as indices before their parents), and `steps`, the
// memo after each step run (`stage0`: the frontends' DAG hash-consed,
// groups without candidates; `pass1`; `pass2_local_window_composition`:
// local window composition's tumbling-window candidates; `pass2`: Pass
// 2's output, the memo Stage 2 implements). Each
// groups without candidates; `pass1`; `pass2`: Pass 2's output, the memo
// Stage 2 implements). Each
// step lists its groups (`id`, `label`, `children`, `parents`,
// `queries`, `roots_of`, `node`) and their candidates (`index`,
// `implements`, `children`, `origin`, `accuracy_budget_parts`, `added`
Expand All @@ -102,8 +101,8 @@ use asap_devtools::planning_stages::{
use asap_devtools::planning_stages_text::planning_stages_text;
use asap_devtools::SqlCatalog;
use asap_frontend_promql::{metric_type_warnings, promql_metric_names};
use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection};
use asap_planner::{installable_plan_of, PlanningModels, SelectionError};
use asap_stage3_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection};
use asap_types::accuracy::{default_metric_for_root, AccuracyTarget};
use asap_types::deployment_capabilities::DeploymentCapabilities;
use asap_types::ir::flat::{flatten, FlatDag, FlatNode};
Expand Down Expand Up @@ -380,13 +379,6 @@ fn memo_json(run: &PlanningStagesRun) -> Value {
if let Some(pass1) = &run.run.pass1_memo {
named.push(("pass1", "Stage 1, Pass 1", pass1));
}
if let Some(pass2) = &run.pass2_local_window_composition {
named.push((
"pass2_local_window_composition",
"Stage 1, Pass 2: local window composition",
pass2,
));
}
if let Some(logical) = &run.run.logical_asap_candidates {
named.push(("pass2", "Stage 1, Pass 2", &logical.memo));
}
Expand Down
4 changes: 2 additions & 2 deletions crates/devtools/src/bin/sketch_coverage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ use asap_devtools::corpora::{
};
use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog};
use asap_frontend_sql::lower_sql_dialect;
use asap_logical_asap_optimization::memo_of_queries;
use asap_logical_asap_optimization::pass1_memo;
use asap_stage0_memo::memo_of_queries;
use asap_stage1_pass1_per_computation_candidates::pass1_memo;
use asap_types::ir::schema::FieldDataType;
use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot};
use asap_types::workload::SqlDialect;
Expand Down
26 changes: 6 additions & 20 deletions crates/devtools/src/planning_stages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@
//! [`run_stages_until`] for `asap_planner`.

use asap_frontend_sql::SqlCatalog;
use asap_logical_asap_optimization::{
memo_of_queries, pass1_memo, pass2_after_local_window_composition,
pass2_local_window_composition, Pass1Error,
};
use asap_physical_asap_optimization::generate_physical_asap_memo_candidates;
use asap_plan_selection::select_plan;
use asap_planner::{
run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectionError,
};
use asap_stage0_memo::memo_of_queries;
use asap_stage1_pass1_per_computation_candidates::{pass1_memo, Pass1Error};
use asap_stage1_pass2_reuse_candidates::generate_logical_asap_memo_candidates;
use asap_stage2_physical_optimization::generate_physical_asap_memo_candidates;
use asap_stage3_plan_selection::select_plan;
use asap_types::ir::schema::series_identity::with_promql_series_identity;
use asap_types::ir::schema::{DataType, Field, Schema};
use asap_types::ir::QueryRoot;
Expand Down Expand Up @@ -70,9 +69,6 @@ pub struct PlanningStagesRun {
/// Stage 0's memo groups (the roots hash-consed), without candidates;
/// built by `run_stages_until` only (`asap_planner` prints them).
pub stage0_memo_groups: Option<Memo>,
/// Stage 1 Pass 2's memo after local window composition (rule 1), before
/// the other rules; `None` before Pass 2.
pub pass2_local_window_composition: Option<Memo>,
/// The stages' outputs: Pass 1, Pass 2, Stage 2 and the selection.
pub run: PlanningRun,
}
Expand Down Expand Up @@ -201,7 +197,6 @@ pub fn run_search_over(
Ok(PlanningStagesRun {
run_until: RunUntil::Stage3,
stage0_memo_groups: None,
pass2_local_window_composition: None,
stage0_query_language_frontends: roots,
run,
})
Expand Down Expand Up @@ -232,7 +227,6 @@ pub fn run_stages_until(
let mut stages = PlanningStagesRun {
run_until: until,
stage0_memo_groups: Some(stage0_dag.clone()),
pass2_local_window_composition: None,
stage0_query_language_frontends: roots.clone(),
run: PlanningRun {
pass1_memo: None,
Expand Down Expand Up @@ -273,15 +267,7 @@ pub fn run_stages_until(
if until == RunUntil::Pass1 {
return Ok(stages);
}
let composed = match pass2_local_window_composition(pass1, entries) {
Ok(composed) => composed,
Err(error) => {
run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string()));
return Ok(stages);
}
};
stages.pass2_local_window_composition = Some(composed.clone());
let logical = match pass2_after_local_window_composition(composed, models.evidence) {
let logical = match generate_logical_asap_memo_candidates(pass1, entries, models.evidence) {
Ok(logical) => logical,
Err(error) => {
run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string()));
Expand Down
12 changes: 2 additions & 10 deletions crates/devtools/src/planning_stages_text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,23 +68,15 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String {
return out;
}

let (Some(pass2), Some(logical)) = (
&stages.pass2_local_window_composition,
&run.logical_asap_candidates,
) else {
let Some(logical) = &run.logical_asap_candidates else {
failure(&mut out);
return out;
};
section(
&mut out,
"Stage 1 Pass 2, after local window composition (tumbling-window candidates)",
);
write_memo(&mut out, pass2, Some(pass1), true);
section(
&mut out,
"Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates",
);
write_memo(&mut out, &logical.memo, Some(pass2), true);
write_memo(&mut out, &logical.memo, Some(pass1), true);
if until == RunUntil::Pass2 {
return out;
}
Expand Down
Loading
Loading