diff --git a/crates/devtools/src/planner_fuzz/relations/mod.rs b/crates/devtools/src/planner_fuzz/relations/mod.rs index b5b6ab13d..f3dae2acf 100644 --- a/crates/devtools/src/planner_fuzz/relations/mod.rs +++ b/crates/devtools/src/planner_fuzz/relations/mod.rs @@ -909,6 +909,7 @@ fn with_operator(node: &OperatorNode, operator: Operator) -> Rc { /// estimate can stand in for it without changing what the nodes above /// compute. Returns its state and statistic. fn root_estimate(root: &Rc) -> Result<(Rc, SketchStatistic), String> { + let root = &column_projections_folded(root); let estimate = match root.non_asap() { Some(NonASAPOp::Project { cols, child, .. }) if cols @@ -931,6 +932,52 @@ fn root_estimate(root: &Rc) -> Result<(Rc, SketchSta } } +/// `root` with a chain of `Project`s of plain column references folded +/// into one: SQL puts a grouped estimate under two (`Project [city, v]` +/// over `Project [v, city]` over the `SummaryEstimate`), and the outer one +/// picks columns of the inner one's output. A root that is not such a +/// chain is returned as it is. +fn column_projections_folded(root: &Rc) -> Rc { + let plain_columns = |cols: &[ProjectItem]| { + cols.iter() + .map(|item| match item.expr { + ScalarExpr::Column(index) => Some(index), + _ => None, + }) + .collect::>>() + }; + let Some(NonASAPOp::Project { cols, child }) = root.non_asap() else { + return root.clone(); + }; + let child = column_projections_folded(child); + let ( + Some(outer), + Some(NonASAPOp::Project { + cols: inner_cols, + child: grandchild, + }), + ) = (plain_columns(cols), child.non_asap()) + else { + return root.clone(); + }; + let Some(inner) = plain_columns(inner_cols) else { + return root.clone(); + }; + let folded = outer + .iter() + .zip(cols) + .map(|(&index, item)| ProjectItem { + alias: item.alias.clone(), + expr: ScalarExpr::Column(inner[index]), + }) + .collect(); + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { + cols: folded, + child: grandchild.clone(), + })) + .unwrap_or_else(|_| root.clone()) +} + /// `root` with its root estimate ([`root_estimate`]) replaced by /// `estimate`; a root `Project` keeps its columns by name, except the /// `dropped` group labels (a rollup's), which it loses. Any other column @@ -940,6 +987,7 @@ fn with_root_estimate( estimate: Rc, dropped: &[String], ) -> Result, String> { + let root = &column_projections_folded(root); let Some(NonASAPOp::Project { cols, child }) = root.non_asap() else { return Ok(estimate); }; diff --git a/crates/devtools/tests/planner_fuzz.rs b/crates/devtools/tests/planner_fuzz.rs index b16317a64..81668cb41 100644 --- a/crates/devtools/tests/planner_fuzz.rs +++ b/crates/devtools/tests/planner_fuzz.rs @@ -219,20 +219,48 @@ fn witness_of(relation: Relation, workload: FuzzWorkload) -> Witness { } /// The sketch rollup witness builds and costs for a grouped quantile: -/// a `SummaryMerge` with `group_by` over the finer per-group KLL. +/// a `SummaryMerge` with `group_by` over the finer per-group KLL. The +/// query must select a KLL, or there is no sketch to roll up. A quantile +/// selects a sketch only over many rows per group and evaluation: the +/// generated PromQL `quantile by (job, instance)` reads 40 latest samples +/// per group, and #902's p90 per city over 5 min at 666.67 rows/s, so +/// both plan exact. p90 per (city, host) over 1 h at 100,000 rows/s +/// (4 cities x 5 hosts, ε 0.05, δ 0.01) reads a KLL tumbling-window +/// chain (k = 132), and p90 per city rolls up from it. #[test] fn sketch_rollup_witness_costs_for_a_grouped_quantile() { - let q = promql( - "m_gauge", - None, - Some((OuterOp::Quantile(0.9), &["job", "instance"])), - ); - assert_eq!( - witness_of(Relation::CoarserGrouping, generated(q, true)), - Witness::SketchRollup { - labels: vec!["job".into()] + use asap_types::accuracy::AccuracyTarget; + let mut workload = seed("issue902.json"); + let per_city_and_host = SqlSpec { + table: "metrics".into(), + aggregate: SqlAggregate::Quantile(0.9), + group_by: vec!["city".into(), "host".into()], + filters: vec![], + window_ms: 3_600_000, + }; + workload.queries = vec![workload.queries[2] + .with_spec(QuerySpec::Sql(per_city_and_host)) + .with_accuracy(AccuracyTarget::max_error(0.05).failure_probability(0.01))]; + workload.data = serde_json::from_value(serde_json::json!({ + "arrival": {"ingesting": {"rows_per_second": 100_000.0}}, + "sources": {"metrics": { + "series_count": 100, + "distinct_values_per_label": {"city": 4, "host": 5}, + }}, + })) + .expect("the data facts deserialize"); + match check(Relation::CoarserGrouping, workload, 0) { + Outcome::Holds(m) | Outcome::Violated(m) => { + assert!(m.base_plan.contains("Kll"), "{}", m.base_plan); + assert_eq!( + m.witness.and_then(|w| w.derivation), + Some(Witness::SketchRollup { + labels: vec!["city".into()] + }) + ); } - ); + Outcome::Inconclusive { reason } => panic!("CoarserGrouping is inconclusive: {reason}"), + } } /// The exact rollup witness builds and costs for a grouped count (a sum diff --git a/crates/devtools/tests/sketch_plan_snapshots/manifest.json b/crates/devtools/tests/sketch_plan_snapshots/manifest.json index a8f59d22a..edfa735f4 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/manifest.json +++ b/crates/devtools/tests/sketch_plan_snapshots/manifest.json @@ -74,6 +74,37 @@ "covers": "random-mixed seed-862-long workload-01, query 3: DDSketch pane chain", "workload": "crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json", "deployment": "mvp" + }, + { + "id": "sketch-heavy-univmon-l2-mvp", + "covers": "sketch-heavy UnivMon: l2_over_time over 4 value streams at 10 Hz, 6 h, relative 0.05: an ingestion-time UnivMon tumbling-window chain (5 rows x 2,367 columns)", + "workload": "tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-6h-every60s-releps0.05-d0.05.json", + "deployment": "mvp" + }, + { + "id": "sketch-heavy-cmsheap-topk10-mvp", + "covers": "sketch-heavy Count-Min + heap: topk(10) of a rate over 1,000 URLs at L1-normalized 0.01: a query-time CMS heap (depth 10, width 272) over the exact rate chain", + "workload": "tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json", + "deployment": "mvp" + }, + { + "id": "sketch-heavy-csheap-topk10-mvp", + "covers": "sketch-heavy Count Sketch + heap: topk(10) of a rate over 300 URLs at L2-normalized 0.05: a query-time Count Sketch heap (depth 117, width 1,200)", + "workload": "tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-1hz-5m-every60s-l2eps0.05-d0.1.json", + "deployment": "mvp" + }, + { + "id": "sketch-heavy-sql-distinct-by-region-mvp", + "covers": "sketch-heavy KMV: SQL COUNT(DISTINCT user_id) by region at 100,000 rows/s over 1 h: a KMV tumbling-window chain per region (k = 26,656)", + "workload": "tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__100000rps-3600s-every60s-releps0.05-d0.05.json", + "tables": "tools/dag-viewer/workloads/sketch-heavy/tables.json", + "deployment": "mvp" + }, + { + "id": "sketch-heavy-edge-dashboard-mvp", + "covers": "sketch-heavy mixed: top-k, total rate, L2 and distinct over two metrics: CMS heap, exact rate, UnivMon chain (624 columns) and KMV chain in one plan", + "workload": "tools/dag-viewer/workloads/sketch-heavy/mixed-edge-dashboard__u1000-i1-1hz-r4-10hz.json", + "deployment": "mvp" } ] } diff --git a/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-cmsheap-topk10-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-cmsheap-topk10-mvp.json new file mode 100644 index 000000000..86f0f507a --- /dev/null +++ b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-cmsheap-topk10-mvp.json @@ -0,0 +1,982 @@ +{ + "accuracy_guarantees": "enforced", + "materialized_views": [ + { + "coverage": { + "definition_id": "168c8f8a0ff8bdcbd972235723737e94", + "selection": [ + { + "columns": [] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 1, + "refresh": "at_ingestion", + "retention_ms": 300000 + } + ], + "queries": [ + "topk(10, sum by (url) (rate(http_requests_total[5m])))" + ], + "selected": { + "accepted": [ + { + "accepted_metric": "absolute_stream_l1", + "sketches": [ + "CmsWithHeap" + ], + "summary": "accepted absolute_stream_l1 (CmsWithHeap)" + } + ], + "cost": { + "cost_per_second": 3.53185204348e-9, + "cpu_core_seconds_per_second": 0.000225728, + "per_node": { + "0": { + "cost_per_second": 6.52152e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 5.8e-6, + "detail": "scan 1000 samples; x 1.0000/s", + "multiplier": 1, + "operator": "Scan", + "timing": "ingestion_time" + }, + "1": { + "cost_per_second": 2.41050244002e-9, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.000126, + "detail": "build exact Rate accumulator in tumbling windows of 60000 ms for 1000 groups: 1000 rows x 70 ops, 100 group visits, 16.7 states emitted; x 1.0000/s; keeps 864000 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + }, + "2": { + "cost_per_second": 1.03070001518e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 9.16666666667e-7, + "detail": "time range 300s: 5 tumbling windows of states; x 0.0167/s; holds 144 bytes for 0.055 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "3": { + "cost_per_second": 5.34097867254e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0000475, + "detail": "merge 5000 states into 1000; x 0.0167/s; holds 144000 bytes for 2.850 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "4": { + "cost_per_second": 4.685e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0000416666666667, + "detail": "finalize 1000 accumulators; x 0.0167/s", + "multiplier": 1, + "operator": "FinalizeExactAccumulator", + "timing": "query_time" + }, + "5": { + "cost_per_second": 4.25717620533e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 3.78616666667e-6, + "detail": "build CmsWithHeap into 1 states: 1000 rows x 227 ops, 1 group visits; x 0.0167/s; holds 23894 bytes for 0.227 ms", + "multiplier": 1, + "operator": "SummaryAgg", + "timing": "query_time" + }, + "6": { + "cost_per_second": 6.559e-13, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 5.83333333333e-8, + "detail": "estimate 1 groups x 10; x 0.0167/s", + "multiplier": 1, + "operator": "SummaryEstimate", + "timing": "query_time" + }, + "7": { + "cost_per_second": 1.8740000046e-15, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.66666666667e-10, + "detail": "operator: pass 10 rows; x 0.0167/s; holds 24 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + } + }, + "source": "analytical-cost-v2 (illustrative statistics)", + "usage": { + "average_memory_bytes": 864006.930599, + "cpu_ops_per_second": 220928.0, + "disk_bytes": 0, + "peak_cpu_cores": 2, + "peak_memory_bytes": 1032110, + "scanned_bytes_per_second": 48000.0 + } + }, + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Sketch": "CmsWithHeap" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [] + }, + "producer": 5, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "id": 0, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "id": 1, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 0, + "filter": null, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": "PerEntity", + "summary_type": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "tumbling_window_width_ms": 60000 + } + } + }, + "retention_ms": 300000, + "view_coverage": { + "definition_id": "168c8f8a0ff8bdcbd972235723737e94", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 2, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 1, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + } + } + } + }, + { + "id": 3, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 2 + ] + } + } + } + }, + { + "id": 4, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 3 + } + } + } + }, + { + "id": 5, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Sketch": "CmsWithHeap" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "CmsWithHeap": { + "depth": 10, + "heap_size": 10, + "width": 272 + } + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 4, + "filter": null, + "input": { + "item": { + "Column": { + "Named": "url" + } + }, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": { + "Reduce": [] + }, + "summary_type": { + "Sketch": "CmsWithHeap" + } + } + } + } + }, + { + "id": 6, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 5 + } + } + } + }, + { + "id": 7, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + } + ], + "time_index": null, + "unique_keys": [ + [ + 1 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 6, + "cols": [ + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + }, + { + "alias": "url", + "expr": { + "Column": 0 + } + } + ] + } + } + } + } + ], + "roots": [ + 7 + ], + "scalar_roots": [] + } + }, + "warnings": [ + "assumed data fact: the data arrival declares no max_out_of_order_ms, so the samples of source http_requests_total that an ingestion-time rate, increase or irate state reads are taken to arrive in timestamp order; if this does not hold, answers may be wrong by an amount no accuracy target bounds (a late sample can be read as a counter reset), or ingestion may fail" + ] +} diff --git a/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-csheap-topk10-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-csheap-topk10-mvp.json new file mode 100644 index 000000000..a668249ba --- /dev/null +++ b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-csheap-topk10-mvp.json @@ -0,0 +1,982 @@ +{ + "accuracy_guarantees": "enforced", + "materialized_views": [ + { + "coverage": { + "definition_id": "168c8f8a0ff8bdcbd972235723737e94", + "selection": [ + { + "columns": [] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 1, + "refresh": "at_ingestion", + "retention_ms": 300000 + } + ], + "queries": [ + "topk(10, sum by (url) (rate(http_requests_total[5m])))" + ], + "selected": { + "accepted": [ + { + "accepted_metric": "absolute_stream_l2", + "sketches": [ + "CountSketchWithHeap" + ], + "summary": "accepted absolute_stream_l2 (CountSketchWithHeap)" + } + ], + "cost": { + "cost_per_second": 1.16833032935e-9, + "cpu_core_seconds_per_second": 0.0000773913333333, + "per_node": { + "0": { + "cost_per_second": 1.956456e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.74e-6, + "detail": "scan 300 samples; x 1.0000/s", + "multiplier": 1, + "operator": "Scan", + "timing": "ingestion_time" + }, + "1": { + "cost_per_second": 7.23150732005e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0000378, + "detail": "build exact Rate accumulator in tumbling windows of 60000 ms for 300 groups: 300 rows x 70 ops, 30 group visits, 5.0 states emitted; x 1.0000/s; keeps 259200 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + }, + "2": { + "cost_per_second": 3.09210004555e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 2.75e-7, + "detail": "time range 300s: 5 tumbling windows of states; x 0.0167/s; holds 144 bytes for 0.017 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "3": { + "cost_per_second": 1.60227708053e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.00001425, + "detail": "merge 1500 states into 300; x 0.0167/s; holds 43200 bytes for 0.855 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "4": { + "cost_per_second": 1.4055e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0000125, + "detail": "finalize 300 accumulators; x 0.0167/s", + "multiplier": 1, + "operator": "FinalizeExactAccumulator", + "timing": "query_time" + }, + "5": { + "cost_per_second": 1.21087455242e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0000107678333333, + "detail": "build CountSketchWithHeap into 1 states: 300 rows x 2153 ops, 1 group visits; x 0.0167/s; holds 1125334 bytes for 0.646 ms", + "multiplier": 1, + "operator": "SummaryAgg", + "timing": "query_time" + }, + "6": { + "cost_per_second": 6.559e-13, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 5.83333333333e-8, + "detail": "estimate 1 groups x 10; x 0.0167/s", + "multiplier": 1, + "operator": "SummaryEstimate", + "timing": "query_time" + }, + "7": { + "cost_per_second": 1.8740000046e-15, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.66666666667e-10, + "detail": "operator: pass 10 rows; x 0.0167/s; holds 24 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + } + }, + "source": "analytical-cost-v2 (illustrative statistics)", + "usage": { + "average_memory_bytes": 259212.733049, + "cpu_ops_per_second": 75951.3333333, + "disk_bytes": 0, + "peak_cpu_cores": 2, + "peak_memory_bytes": 1427950, + "scanned_bytes_per_second": 14400.0 + } + }, + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Sketch": "CountSketchWithHeap" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [] + }, + "producer": 5, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "id": 0, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "id": 1, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 0, + "filter": null, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": "PerEntity", + "summary_type": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "tumbling_window_width_ms": 60000 + } + } + }, + "retention_ms": 300000, + "view_coverage": { + "definition_id": "168c8f8a0ff8bdcbd972235723737e94", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 2, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 1, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + } + } + } + }, + { + "id": 3, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 2 + ] + } + } + } + }, + { + "id": 4, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 3 + } + } + } + }, + { + "id": 5, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Sketch": "CountSketchWithHeap" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "CountSketchWithHeap": { + "depth": 117, + "heap_size": 10, + "width": 1200 + } + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 4, + "filter": null, + "input": { + "item": { + "Column": { + "Named": "url" + } + }, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": { + "Reduce": [] + }, + "summary_type": { + "Sketch": "CountSketchWithHeap" + } + } + } + } + }, + { + "id": 6, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 5 + } + } + } + }, + { + "id": 7, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + } + ], + "time_index": null, + "unique_keys": [ + [ + 1 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 6, + "cols": [ + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + }, + { + "alias": "url", + "expr": { + "Column": 0 + } + } + ] + } + } + } + } + ], + "roots": [ + 7 + ], + "scalar_roots": [] + } + }, + "warnings": [ + "assumed data fact: the data arrival declares no max_out_of_order_ms, so the samples of source http_requests_total that an ingestion-time rate, increase or irate state reads are taken to arrive in timestamp order; if this does not hold, answers may be wrong by an amount no accuracy target bounds (a late sample can be read as a counter reset), or ingestion may fail" + ] +} diff --git a/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-edge-dashboard-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-edge-dashboard-mvp.json new file mode 100644 index 000000000..aa69c4cc5 --- /dev/null +++ b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-edge-dashboard-mvp.json @@ -0,0 +1,2893 @@ +{ + "accuracy_guarantees": "enforced", + "materialized_views": [], + "materialized_views_error": "the executor cannot refresh materialized view 1 from its inputs: refreshed with views [1, 15]: invalid DAG: per-entity update requires a projected value column", + "queries": [ + "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "sum(rate(http_requests_total[5m]))", + "l2_over_time(dns_query_client[24h])", + "distinct_over_time(dns_query_client[24h])" + ], + "selected": { + "accepted": [ + { + "accepted_metric": "absolute_stream_l1", + "sketches": [ + "CmsWithHeap" + ], + "summary": "accepted absolute_stream_l1 (CmsWithHeap)" + }, + { + "accepted_metric": null, + "sketches": [], + "summary": "accepted exact" + }, + { + "accepted_metric": "relative", + "sketches": [ + "UnivMon" + ], + "summary": "accepted relative (UnivMon)" + }, + { + "accepted_metric": "relative", + "sketches": [ + "Kmv" + ], + "summary": "accepted relative (Kmv)" + } + ], + "cost": { + "cost_per_second": 5.0223721677e-7, + "cpu_core_seconds_per_second": 0.0242945297659, + "per_node": { + "0": { + "cost_per_second": 6.52152e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 5.8e-6, + "detail": "scan 1000 samples; x 1.0000/s", + "multiplier": 1, + "operator": "Scan", + "timing": "ingestion_time" + }, + "1": { + "cost_per_second": 1.15199548068e-8, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.000126, + "detail": "build exact Rate accumulator in tumbling windows of 60000 ms for 1000 groups: 1000 rows x 70 ops, 100 group visits, 16.7 states emitted; x 1.0000/s; keeps 8784000 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + }, + "10": { + "cost_per_second": 3.71052004858e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.000033, + "detail": "time range 300s: 30 tumbling windows of states; x 0.1000/s; holds 128 bytes for 0.330 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "11": { + "cost_per_second": 1.30432107792e-8, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.00116, + "detail": "merge 30000 states into 1000; x 0.1000/s; holds 128000 bytes for 11.600 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "12": { + "cost_per_second": 2.811e-9, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.00025, + "detail": "finalize 1000 accumulators; x 0.1000/s", + "multiplier": 1, + "operator": "FinalizeExactAccumulator", + "timing": "query_time" + }, + "13": { + "cost_per_second": 7.49974801841e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 6.67e-7, + "detail": "hash aggregate 1000 rows into 1 groups: 4 ops per row, 1 group visits; x 0.1000/s; holds 24 bytes for 0.007 ms", + "multiplier": 1, + "operator": "Aggregate", + "timing": "query_time" + }, + "14": { + "cost_per_second": 1.888992e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.68e-7, + "detail": "scan 40 samples; x 1.0000/s", + "multiplier": 1, + "operator": "Scan", + "timing": "ingestion_time" + }, + "15": { + "cost_per_second": 1.70731182854e-7, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 6.27466666667e-6, + "detail": "build UnivMon in tumbling windows of 60000 ms for 4 groups: 40 rows x 137 ops, 0 group visits, 0.1 states emitted; x 1.0000/s; keeps 148376888 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + }, + "16": { + "cost_per_second": 1.18736952661e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.056e-6, + "detail": "time range 86400s: 1440 tumbling windows of states; x 0.0167/s; holds 25742 bytes for 0.063 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "17": { + "cost_per_second": 2.79582006382e-8, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.00248647333333, + "detail": "merge 5760 states into 4; x 0.0167/s; holds 102968 bytes for 149.188 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "18": { + "cost_per_second": 2.6236e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 2.33333333333e-7, + "detail": "estimate 4 groups x 1; x 0.0167/s", + "multiplier": 1, + "operator": "SummaryEstimate", + "timing": "query_time" + }, + "19": { + "cost_per_second": 7.49600002454e-16, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 6.66666666667e-11, + "detail": "operator: pass 4 rows; x 0.0167/s; holds 32 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + }, + "2": { + "cost_per_second": 1.23684001822e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.000011, + "detail": "time range 3600s: 60 tumbling windows of states; x 0.0167/s; holds 144 bytes for 0.660 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "20": { + "cost_per_second": 4.40916325605e-8, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0000332613333333, + "detail": "build Kmv in tumbling windows of 300000 ms for 4 groups: 40 rows x 815 ops, 0 group visits, 0.0 states emitted; x 1.0000/s; keeps 38009280 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + }, + "21": { + "cost_per_second": 4.74948157434e-13, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 4.224e-8, + "detail": "time range 86400s: 288 tumbling windows of states; x 0.0033/s; holds 32880 bytes for 0.013 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "22": { + "cost_per_second": 2.19826498329e-7, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0195484624459, + "detail": "merge 1152 states into 4; x 0.0033/s; holds 1049024 bytes for 5864.539 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "23": { + "cost_per_second": 5.2472e-13, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 4.66666666667e-8, + "detail": "estimate 4 groups x 1; x 0.0033/s", + "multiplier": 1, + "operator": "SummaryEstimate", + "timing": "query_time" + }, + "24": { + "cost_per_second": 1.49920000491e-16, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.33333333333e-11, + "detail": "operator: pass 4 rows; x 0.0033/s; holds 32 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + }, + "3": { + "cost_per_second": 4.14160100573e-9, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.000368333333333, + "detail": "merge 60000 states into 1000; x 0.0167/s; holds 144000 bytes for 22.100 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "4": { + "cost_per_second": 4.685e-10, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.0000416666666667, + "detail": "finalize 1000 accumulators; x 0.0167/s", + "multiplier": 1, + "operator": "FinalizeExactAccumulator", + "timing": "query_time" + }, + "5": { + "cost_per_second": 4.25717620533e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 3.78616666667e-6, + "detail": "build CmsWithHeap into 1 states: 1000 rows x 227 ops, 1 group visits; x 0.0167/s; holds 23894 bytes for 0.227 ms", + "multiplier": 1, + "operator": "SummaryAgg", + "timing": "query_time" + }, + "6": { + "cost_per_second": 6.559e-13, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 5.83333333333e-8, + "detail": "estimate 1 groups x 10; x 0.0167/s", + "multiplier": 1, + "operator": "SummaryEstimate", + "timing": "query_time" + }, + "7": { + "cost_per_second": 1.8740000046e-15, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.66666666667e-10, + "detail": "operator: pass 10 rows; x 0.0167/s; holds 24 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + }, + "8": { + "cost_per_second": 4.72248e-11, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 4.2e-6, + "detail": "scan 1000 samples; x 1.0000/s", + "multiplier": 1, + "operator": "Scan", + "timing": "ingestion_time" + }, + "9": { + "cost_per_second": 6.97014365045e-9, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.000214, + "detail": "build exact Rate accumulator in tumbling windows of 10000 ms for 1000 groups: 1000 rows x 33 ops, 100 group visits, 100.0 states emitted; x 1.0000/s; keeps 3968000 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + } + }, + "source": "analytical-cost-v2 (illustrative statistics)", + "usage": { + "average_memory_bytes": 199159132.478, + "cpu_ops_per_second": 24286401.7659, + "disk_bytes": 0, + "peak_cpu_cores": 5, + "peak_memory_bytes": 200645172, + "scanned_bytes_per_second": 81280.0 + } + }, + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Sketch": "CmsWithHeap" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [] + }, + "producer": 5, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 13, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 12, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 15, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 14, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 16, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 15, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 17, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 16, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 18, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 17, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 19, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "frequency_l2", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 18, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 20, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 14, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 21, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 20, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 22, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 21, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 23, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 22, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 24, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "cardinality", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 23, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "id": 0, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "id": 1, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 0, + "filter": null, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": "PerEntity", + "summary_type": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "tumbling_window_width_ms": 60000 + } + } + }, + "retention_ms": 3600000, + "view_coverage": { + "definition_id": "168c8f8a0ff8bdcbd972235723737e94", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 2, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2, + 3 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 1, + "kind": "range", + "range": { + "nanos": 0, + "secs": 3600 + } + } + } + } + }, + { + "id": 3, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 2 + ] + } + } + } + }, + { + "id": 4, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 3 + } + } + } + }, + { + "id": 5, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Sketch": "CmsWithHeap" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "CmsWithHeap": { + "depth": 10, + "heap_size": 10, + "width": 272 + } + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 4, + "filter": null, + "input": { + "item": { + "Column": { + "Named": "url" + } + }, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": { + "Reduce": [] + }, + "summary_type": { + "Sketch": "CmsWithHeap" + } + } + } + } + }, + { + "id": 6, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 5 + } + } + } + }, + { + "id": 7, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": true + } + ], + "time_index": null, + "unique_keys": [ + [ + 1 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 6, + "cols": [ + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + }, + { + "alias": "url", + "expr": { + "Column": 0 + } + } + ] + } + } + } + }, + { + "id": 8, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "id": 9, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 8, + "filter": null, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": "PerEntity", + "summary_type": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "tumbling_window_width_ms": 10000 + } + } + }, + "retention_ms": 300000, + "view_coverage": { + "definition_id": "d6d9c0dd22016af382a77ac30f6c9597", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 10, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 9, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + } + } + } + }, + { + "id": 11, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 10 + ] + } + } + } + }, + { + "id": 12, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 11 + } + } + } + }, + { + "id": 13, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Aggregate": { + "child": 12, + "filters": [], + "having": null, + "measures": [ + { + "col": null, + "kind": "sum" + } + ], + "output_names": [ + "" + ], + "reduction": { + "Reduce": [] + } + } + } + } + }, + { + "id": 14, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "dns_query_client" + } + } + } + } + } + }, + { + "id": 15, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "parameters": { + "UnivMon": { + "heap_size": 1, + "layers": 1, + "sketch_cols": 624, + "sketch_rows": 5 + } + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 14, + "filter": null, + "input": { + "item": { + "Column": "SampleValue" + }, + "weight": { + "Constant": 1.0 + } + }, + "reduction": "PerEntity", + "summary_type": { + "Sketch": "UnivMon" + }, + "tumbling_window_width_ms": 60000 + } + } + }, + "retention_ms": 86400000, + "view_coverage": { + "definition_id": "fc4b8ac4c3900ad21165396ade1e3713", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 16, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "UnivMon": { + "heap_size": 1, + "layers": 1, + "sketch_cols": 624, + "sketch_rows": 5 + } + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 15, + "kind": "range", + "range": { + "nanos": 0, + "secs": 86400 + } + } + } + } + }, + { + "id": 17, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "UnivMon": { + "heap_size": 1, + "layers": 1, + "sketch_cols": 624, + "sketch_rows": 5 + } + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 16 + ] + } + } + } + }, + { + "id": 18, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "frequency_l2", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": "FrequencyL2", + "summary_input": 17 + } + } + } + }, + { + "id": 19, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 18, + "cols": [ + { + "alias": "ts", + "expr": { + "Column": 0 + } + }, + { + "alias": "value", + "expr": { + "Column": 1 + } + }, + { + "alias": "$promql_series_identity", + "expr": { + "Column": 2 + } + } + ] + } + } + } + }, + { + "id": 20, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "parameters": { + "Kmv": { + "k": 26656 + } + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 14, + "filter": null, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + } + }, + "reduction": "PerEntity", + "summary_type": { + "Sketch": "Kmv" + }, + "tumbling_window_width_ms": 300000 + } + } + }, + "retention_ms": 86400000, + "view_coverage": { + "definition_id": "a47f127e227fff5d662b55cb0d4d7dcf", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 21, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "Kmv": { + "k": 26656 + } + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 20, + "kind": "range", + "range": { + "nanos": 0, + "secs": 86400 + } + } + } + } + }, + { + "id": 22, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "Kmv": { + "k": 26656 + } + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 21 + ] + } + } + } + }, + { + "id": 23, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "cardinality", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": "Cardinality", + "summary_input": 22 + } + } + } + }, + { + "id": 24, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 23, + "cols": [ + { + "alias": "ts", + "expr": { + "Column": 0 + } + }, + { + "alias": "value", + "expr": { + "Column": 1 + } + }, + { + "alias": "$promql_series_identity", + "expr": { + "Column": 2 + } + } + ] + } + } + } + } + ], + "roots": [ + 7, + 13, + 19, + 24 + ], + "scalar_roots": [] + } + }, + "warnings": [ + "assumed data fact: the data arrival declares no max_out_of_order_ms, so the samples of source http_requests_total that an ingestion-time rate, increase or irate state reads are taken to arrive in timestamp order; if this does not hold, answers may be wrong by an amount no accuracy target bounds (a late sample can be read as a counter reset), or ingestion may fail" + ] +} diff --git a/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-sql-distinct-by-region-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-sql-distinct-by-region-mvp.json new file mode 100644 index 000000000..0baee5f66 --- /dev/null +++ b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-sql-distinct-by-region-mvp.json @@ -0,0 +1,856 @@ +{ + "accuracy_guarantees": "enforced", + "materialized_views": [ + { + "coverage": { + "definition_id": "94b0d2a5956989628f8bd3961df3e17f", + "selection": [ + { + "columns": [] + } + ] + }, + "inputs": { + "materialized_views": [] + }, + "node_id": 1, + "refresh": "at_ingestion", + "retention_ms": 3600000 + } + ], + "queries": [ + "SELECT region, COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY region" + ], + "selected": { + "accepted": [ + { + "accepted_metric": "relative", + "sketches": [ + "Kmv" + ], + "summary": "accepted relative (Kmv)" + } + ], + "cost": { + "cost_per_second": 4.88047463017e-6, + "cpu_core_seconds_per_second": 0.420912952941, + "per_node": { + "0": { + "cost_per_second": 1.191864e-8, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.00106, + "detail": "scan 100000 samples; x 1.0000/s", + "multiplier": 1, + "operator": "Scan", + "timing": "ingestion_time" + }, + "1": { + "cost_per_second": 2.28239135138e-6, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.189898324171, + "detail": "build Kmv in tumbling windows of 60000 ms for 8 groups: 100000 rows x 1884.9785483785768 ops, 1 group visits, 0.1 states emitted; x 1.0000/s; keeps 127957504 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + }, + "2": { + "cost_per_second": 9.89498539681e-13, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 8.8e-8, + "detail": "time range 3600s: 60 tumbling windows of states; x 0.0167/s; holds 262208 bytes for 0.005 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "3": { + "cost_per_second": 2.58615839909e-6, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.229954073837, + "detail": "merge 480 states into 8; x 0.0167/s; holds 2097600 bytes for 13797.244 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "4": { + "cost_per_second": 5.2472e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 4.66666666667e-7, + "detail": "estimate 8 groups x 1; x 0.0167/s", + "multiplier": 1, + "operator": "SummaryEstimate", + "timing": "query_time" + }, + "5": { + "cost_per_second": 1.49920000368e-15, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.33333333333e-10, + "detail": "operator: pass 8 rows; x 0.0167/s; holds 24 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + }, + "6": { + "cost_per_second": 1.49920000368e-15, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.33333333333e-10, + "detail": "operator: pass 8 rows; x 0.0167/s; holds 24 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + } + }, + "source": "analytical-cost-v2 (illustrative statistics)", + "usage": { + "average_memory_bytes": 128439855.688, + "cpu_ops_per_second": 419952952.941, + "disk_bytes": 0, + "peak_cpu_cores": 2, + "peak_memory_bytes": 130317456, + "scanned_bytes_per_second": 9600000.0 + } + }, + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "user_id", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "client_ip", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "bytes", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 1 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 1 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "cardinality", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(DISTINCT clicks.user_id)", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 5, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "id": 0, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "user_id", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "client_ip", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "bytes", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "url", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "user_id", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "client_ip", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "bytes", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "clicks" + } + } + } + } + } + }, + { + "id": 1, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 1 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "parameters": { + "Kmv": { + "k": 26656 + } + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 0, + "filter": null, + "input": { + "item": null, + "weight": { + "Column": { + "Named": "user_id" + } + } + }, + "reduction": { + "Reduce": [ + 4 + ] + }, + "summary_type": { + "Sketch": "Kmv" + }, + "tumbling_window_width_ms": 60000 + } + } + }, + "retention_ms": 3600000, + "view_coverage": { + "definition_id": "94b0d2a5956989628f8bd3961df3e17f", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 2, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 1 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "Kmv": { + "k": 26656 + } + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 1, + "kind": "range", + "range": { + "nanos": 0, + "secs": 3600 + } + } + } + } + }, + { + "id": 3, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Sketch": "Kmv" + }, + "name": "state", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "Kmv": { + "k": 26656 + } + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 2 + ] + } + } + } + }, + { + "id": 4, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "cardinality", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": "Cardinality", + "summary_input": 3 + } + } + } + }, + { + "id": 5, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(DISTINCT clicks.user_id)", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 4, + "cols": [ + { + "alias": "region", + "expr": { + "Column": 0 + } + }, + { + "alias": "count(DISTINCT clicks.user_id)", + "expr": { + "Column": 1 + } + } + ] + } + } + } + }, + { + "id": 6, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(DISTINCT clicks.user_id)", + "nullable": false + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 5, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": null, + "expr": { + "Column": 1 + } + } + ] + } + } + } + } + ], + "roots": [ + 6 + ], + "scalar_roots": [] + } + } +} diff --git a/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-univmon-l2-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-univmon-l2-mvp.json new file mode 100644 index 000000000..58e5c8348 --- /dev/null +++ b/crates/devtools/tests/sketch_plan_snapshots/sketch-heavy-univmon-l2-mvp.json @@ -0,0 +1,727 @@ +{ + "accuracy_guarantees": "enforced", + "materialized_views": [], + "materialized_views_error": "the executor cannot refresh materialized view 1 from its inputs: invalid DAG: per-entity update requires a projected value column", + "queries": [ + "l2_over_time(dns_query_client[6h])" + ], + "selected": { + "accepted": [ + { + "accepted_metric": "relative", + "sketches": [ + "UnivMon" + ], + "summary": "accepted relative (UnivMon)" + } + ], + "cost": { + "cost_per_second": 1.84432972909e-7, + "cpu_core_seconds_per_second": 0.0023018934, + "per_node": { + "0": { + "cost_per_second": 1.888992e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 1.68e-7, + "detail": "scan 40 samples; x 1.0000/s", + "multiplier": 1, + "operator": "Scan", + "timing": "ingestion_time" + }, + "1": { + "cost_per_second": 1.5862002791e-7, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 6.27466666667e-6, + "detail": "build UnivMon in tumbling windows of 60000 ms for 4 groups: 40 rows x 137 ops, 0 group visits, 0.1 states emitted; x 1.0000/s; keeps 137847128 bytes", + "multiplier": 1, + "operator": "SummaryTumblingWindowChain", + "timing": "ingestion_time" + }, + "2": { + "cost_per_second": 2.96844498688e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 2.64e-7, + "detail": "time range 21600s: 360 tumbling windows of states; x 0.0167/s; holds 95462 bytes for 0.016 ms", + "multiplier": 1, + "operator": "TimeRange", + "timing": "query_time" + }, + "3": { + "cost_per_second": 2.58054632125e-8, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 0.00229495333333, + "detail": "merge 1440 states into 4; x 0.0167/s; holds 381848 bytes for 137.697 ms", + "multiplier": 1, + "operator": "SummaryMerge", + "timing": "query_time" + }, + "4": { + "cost_per_second": 2.62360003435e-12, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 2.33333333333e-7, + "detail": "estimate 4 groups x 1; x 0.0167/s; holds 128 bytes for 0.014 ms", + "multiplier": 1, + "operator": "SummaryEstimate", + "timing": "query_time" + }, + "5": { + "cost_per_second": 7.49600009815e-16, + "cost_source": "analytical", + "cpu_core_seconds_per_second": 6.66666666667e-11, + "detail": "operator: pass 4 rows; x 0.0167/s; holds 128 bytes for 0.000 ms", + "multiplier": 1, + "operator": "Project", + "timing": "query_time" + } + }, + "source": "analytical-cost-v2 (illustrative statistics)", + "usage": { + "average_memory_bytes": 137848004.349, + "cpu_ops_per_second": 2301765.4, + "disk_bytes": 0, + "peak_cpu_cores": 2, + "peak_memory_bytes": 138324726, + "scanned_bytes_per_second": 1280.0 + } + }, + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "frequency_l2", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "id": 0, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "ingestion_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "dns_query_client" + } + } + } + } + } + }, + { + "id": 1, + "lifetime": { + "AcrossEvaluations": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "ingestion_time" + }, + "parameters": { + "UnivMon": { + "heap_size": 1, + "layers": 1, + "sketch_cols": 2367, + "sketch_rows": 5 + } + }, + "payload": { + "ASAP": { + "SummaryTumblingWindowChain": { + "child": 0, + "filter": null, + "input": { + "item": { + "Column": "SampleValue" + }, + "weight": { + "Constant": 1.0 + } + }, + "reduction": "PerEntity", + "summary_type": { + "Sketch": "UnivMon" + }, + "tumbling_window_width_ms": 60000 + } + } + }, + "retention_ms": 21600000, + "view_coverage": { + "definition_id": "fc4b8ac4c3900ad21165396ade1e3713", + "selection": [ + { + "columns": [] + } + ] + } + }, + { + "id": 2, + "lifetime": "None", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "window_end", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 0, + 2 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "UnivMon": { + "heap_size": 1, + "layers": 1, + "sketch_cols": 2367, + "sketch_rows": 5 + } + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 1, + "kind": "range", + "range": { + "nanos": 0, + "secs": 21600 + } + } + } + } + }, + { + "id": 3, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Sketch": "UnivMon" + }, + "name": "state", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "parameters": { + "UnivMon": { + "heap_size": 1, + "layers": 1, + "sketch_cols": 2367, + "sketch_rows": 5 + } + }, + "payload": { + "ASAP": { + "SummaryMerge": { + "children": [ + 2 + ] + } + } + } + }, + { + "id": 4, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "frequency_l2", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": "FrequencyL2", + "summary_input": 3 + } + } + } + }, + { + "id": 5, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 4, + "cols": [ + { + "alias": "ts", + "expr": { + "Column": 0 + } + }, + { + "alias": "value", + "expr": { + "Column": 1 + } + }, + { + "alias": "$promql_series_identity", + "expr": { + "Column": 2 + } + } + ] + } + } + } + } + ], + "roots": [ + 5 + ], + "scalar_roots": [] + } + } +} diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index 3db1fb906..cd66eecc7 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -179,6 +179,13 @@ planned as the MVP deployment: raw data not retained, Fargate costs, accuracy guarantees enforced. Stage 3 searches exactly, so planning one of them can take several minutes. +The sketch-heavy set (75 workloads, PromQL and SQL) is synthetic. Each +workload is built so that one sketch family is the cheapest plan that +meets its accuracy target (UnivMon, Count-Min or Count Sketch with a heap, +KMV), or sits next to such a workload as a control. The `blocked-*` +workloads are shapes no sketch serves yet. `workloads/sketch-heavy/gen.py` +writes the files, their `tables.json` and the set's index entry. + `memo-examples/` holds smaller `PlanningWorkload` files, each the smallest subset of an ASAPQuery paper workload template that shows one kind of sharing; open one with **Open stage document or workload…**. diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index 852b0c359..467d19f96 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -641,7 +641,7 @@ def test_index_names_existing_files(self): self.assertNotIn("document", entry) self.assertEqual(files, {path for path in WORKLOADS.glob("*/*.json") if path.name != server.TABLES_FILE and path.parent.name != "planner-layering"}) - self.assertEqual(len(files), 30) + self.assertEqual(len(files), 105) class PageTests(unittest.TestCase): diff --git a/tools/dag-viewer/workloads/index.json b/tools/dag-viewer/workloads/index.json index f639a048a..986bf13a3 100644 --- a/tools/dag-viewer/workloads/index.json +++ b/tools/dag-viewer/workloads/index.json @@ -492,6 +492,315 @@ ], "cost_model": "aws-fargate-us-east-1", "accuracy_guarantees": "enforced" + }, + { + "id": "sketch-heavy", + "name": "Sketch-heavy", + "description": "Workloads where a sketch family is the cheapest plan that meets the accuracy target, with controls next to them. heavy-hitters: topk(10, sum by (url) (rate(http_requests_total[W]))) over 100 to 10,000 URLs at L1- or L2-normalized targets (Count-Min or Count Sketch with a heap). frequency-moments: l2_over_time over 1 to 16 value streams (UnivMon). distinct: SQL COUNT(DISTINCT user_id) at 100,000 or 2,000,000 rows/s and PromQL distinct_over_time (KMV). mixed: several queries that could share one sketch. blocked: SQL top-k, L2, entropy, point counts and HLL-shaped distinct counts, which no sketch serves yet.", + "source": "Synthetic, written by gen.py in this directory (metric, table and label names are made up). Each workload declares only the data facts the shape implies: series count, sample interval, metric type, distinct values per grouping label and column signs; no stream norms. SQL workloads read the clicks table in tables.json. Planned with the default deployment (raw data retained) and cost model, accuracy guarantees enforced.", + "accuracy_guarantees": "enforced", + "workloads": [ + { + "id": "heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every10s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every10s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u1000-i1-10hz-1h-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-1h-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u1000-i1-10hz-24h-every300s-l1eps0.002-d0.01", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-24h-every300s-l1eps0.002-d0.01.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u2500-i1-1hz-1h-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-1hz-1h-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u2500-i1-10hz-5m-every10s-l1eps0.002-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-10hz-5m-every10s-l1eps0.002-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u5000-i1-1hz-5m-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u5000-i1-10hz-1h-every10s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-10hz-1h-every10s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u5000-i1-1hz-24h-every300s-l1eps0.002-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-24h-every300s-l1eps0.002-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u1000-i2-1hz-1h-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i2-1hz-1h-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u2500-i2-10hz-5m-every10s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-10hz-5m-every10s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-cmsheap-topk10__u2500-i2-1hz-24h-every60s-l1eps0.002-d0.05", + "file": "sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-1hz-24h-every60s-l1eps0.002-d0.05.json" + }, + { + "id": "heavy-hitters-csheap-topk10__u100-i1-1hz-5m-every60s-l2eps0.1-d0.1", + "file": "sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-5m-every60s-l2eps0.1-d0.1.json" + }, + { + "id": "heavy-hitters-csheap-topk10__u100-i1-10hz-5m-every10s-l2eps0.05-d0.1", + "file": "sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-10hz-5m-every10s-l2eps0.05-d0.1.json" + }, + { + "id": "heavy-hitters-csheap-topk10__u100-i1-1hz-1h-every60s-l2eps0.02-d0.1", + "file": "sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-1h-every60s-l2eps0.02-d0.1.json" + }, + { + "id": "heavy-hitters-csheap-topk10__u300-i1-1hz-5m-every60s-l2eps0.05-d0.1", + "file": "sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-1hz-5m-every60s-l2eps0.05-d0.1.json" + }, + { + "id": "heavy-hitters-csheap-topk10__u300-i1-10hz-1h-every60s-l2eps0.1-d0.1", + "file": "sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-10hz-1h-every60s-l2eps0.1-d0.1.json" + }, + { + "id": "heavy-hitters-csheap-topk10__u1000-i1-1hz-5m-every60s-l2eps0.05-d0.1", + "file": "sketch-heavy/heavy-hitters-csheap-topk10__u1000-i1-1hz-5m-every60s-l2eps0.05-d0.1.json" + }, + { + "id": "heavy-hitters-control-topk10-10-instances-per-url__u1000-i10-10hz-1h-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-control-topk10-10-instances-per-url__u1000-i10-10hz-1h-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-control-topk10-sum-over-time__u1000-i1-1hz-1h-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-control-topk10-sum-over-time__u1000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "heavy-hitters-control-topk10-delta-too-small__u5000-i1-1hz-1h-every60s-l1eps0.01-d0.01", + "file": "sketch-heavy/heavy-hitters-control-topk10-delta-too-small__u5000-i1-1hz-1h-every60s-l1eps0.01-d0.01.json" + }, + { + "id": "heavy-hitters-control-topk10-10000-urls__u10000-i1-1hz-1h-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/heavy-hitters-control-topk10-10000-urls__u10000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r1-1hz-1h-every60s-releps0.1-d0.05", + "file": "sketch-heavy/frequency-moments-univmon-l2__r1-1hz-1h-every60s-releps0.1-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r1-10hz-24h-every60s-releps0.05-d0.05", + "file": "sketch-heavy/frequency-moments-univmon-l2__r1-10hz-24h-every60s-releps0.05-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r1-10hz-6h-every10s-releps0.2-d0.05", + "file": "sketch-heavy/frequency-moments-univmon-l2__r1-10hz-6h-every10s-releps0.2-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r2-10hz-24h-every300s-releps0.1-d0.05", + "file": "sketch-heavy/frequency-moments-univmon-l2__r2-10hz-24h-every300s-releps0.1-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r4-1hz-1h-every10s-releps0.1-d0.05", + "file": "sketch-heavy/frequency-moments-univmon-l2__r4-1hz-1h-every10s-releps0.1-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r4-10hz-6h-every60s-releps0.05-d0.05", + "file": "sketch-heavy/frequency-moments-univmon-l2__r4-10hz-6h-every60s-releps0.05-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r4-10hz-24h-every300s-releps0.1-d0.1", + "file": "sketch-heavy/frequency-moments-univmon-l2__r4-10hz-24h-every300s-releps0.1-d0.1.json" + }, + { + "id": "frequency-moments-univmon-l2__r8-1hz-24h-every60s-releps0.1-d0.1", + "file": "sketch-heavy/frequency-moments-univmon-l2__r8-1hz-24h-every60s-releps0.1-d0.1.json" + }, + { + "id": "frequency-moments-univmon-l2__r8-10hz-24h-every10s-releps0.2-d0.1", + "file": "sketch-heavy/frequency-moments-univmon-l2__r8-10hz-24h-every10s-releps0.2-d0.1.json" + }, + { + "id": "frequency-moments-univmon-l2__r8-10hz-1h-every60s-releps0.05-d0.1", + "file": "sketch-heavy/frequency-moments-univmon-l2__r8-10hz-1h-every60s-releps0.05-d0.1.json" + }, + { + "id": "frequency-moments-univmon-l2__r2-1hz-5m-every60s-releps0.1-d0.05", + "file": "sketch-heavy/frequency-moments-univmon-l2__r2-1hz-5m-every60s-releps0.1-d0.05.json" + }, + { + "id": "frequency-moments-univmon-l2__r8-10hz-5m-every10s-releps0.1-d0.1", + "file": "sketch-heavy/frequency-moments-univmon-l2__r8-10hz-5m-every10s-releps0.1-d0.1.json" + }, + { + "id": "frequency-moments-control-l2-16-resolvers__r16-10hz-24h-every60s-releps0.1-d0.1", + "file": "sketch-heavy/frequency-moments-control-l2-16-resolvers__r16-10hz-24h-every60s-releps0.1-d0.1.json" + }, + { + "id": "distinct-sql-distinct-users__100000rps-3600s-every60s-releps0.05-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users__100000rps-3600s-every60s-releps0.05-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users__100000rps-86400s-every300s-releps0.02-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users__100000rps-86400s-every300s-releps0.02-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users__2000000rps-3600s-every10s-releps0.05-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users__2000000rps-3600s-every10s-releps0.05-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users__2000000rps-86400s-every60s-releps0.02-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users__2000000rps-86400s-every60s-releps0.02-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users-by-region__100000rps-3600s-every60s-releps0.05-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users-by-region__100000rps-3600s-every60s-releps0.05-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users-by-region__2000000rps-86400s-every300s-releps0.05-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-86400s-every300s-releps0.05-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users-by-region__2000000rps-3600s-every60s-releps0.02-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-3600s-every60s-releps0.02-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users-by-service__100000rps-86400s-every300s-releps0.05-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users-by-service__100000rps-86400s-every300s-releps0.05-d0.05.json" + }, + { + "id": "distinct-sql-distinct-users-by-service__2000000rps-3600s-every60s-releps0.05-d0.05", + "file": "sketch-heavy/distinct-sql-distinct-users-by-service__2000000rps-3600s-every60s-releps0.05-d0.05.json" + }, + { + "id": "distinct-promql-distinct-clients__r1-10hz-24h-every60s-releps0.05-d0.1", + "file": "sketch-heavy/distinct-promql-distinct-clients__r1-10hz-24h-every60s-releps0.05-d0.1.json" + }, + { + "id": "distinct-promql-distinct-clients__r4-10hz-6h-every60s-releps0.05-d0.1", + "file": "sketch-heavy/distinct-promql-distinct-clients__r4-10hz-6h-every60s-releps0.05-d0.1.json" + }, + { + "id": "distinct-promql-distinct-clients__r8-1hz-24h-every300s-releps0.02-d0.1", + "file": "sketch-heavy/distinct-promql-distinct-clients__r8-1hz-24h-every300s-releps0.02-d0.1.json" + }, + { + "id": "mixed-topk-and-total__u1000-i1-1hz-1h-every60s", + "file": "sketch-heavy/mixed-topk-and-total__u1000-i1-1hz-1h-every60s.json" + }, + { + "id": "mixed-topk-dashboard-and-alert__u1000-i1-1hz-1h-every10s-and-60s", + "file": "sketch-heavy/mixed-topk-dashboard-and-alert__u1000-i1-1hz-1h-every10s-and-60s.json" + }, + { + "id": "mixed-topk-by-url-and-by-instance__u100-i10-1hz-1h-every60s", + "file": "sketch-heavy/mixed-topk-by-url-and-by-instance__u100-i10-1hz-1h-every60s.json" + }, + { + "id": "mixed-control-topk10-and-topk50__u1000-i1-1hz-1h-every60s", + "file": "sketch-heavy/mixed-control-topk10-and-topk50__u1000-i1-1hz-1h-every60s.json" + }, + { + "id": "mixed-topk-and-total__u5000-i1-1hz-1h-every60s", + "file": "sketch-heavy/mixed-topk-and-total__u5000-i1-1hz-1h-every60s.json" + }, + { + "id": "mixed-topk-dashboard-and-alert__u5000-i1-1hz-1h-every10s-and-60s", + "file": "sketch-heavy/mixed-topk-dashboard-and-alert__u5000-i1-1hz-1h-every10s-and-60s.json" + }, + { + "id": "mixed-topk-by-url-and-by-instance__u500-i10-1hz-1h-every60s", + "file": "sketch-heavy/mixed-topk-by-url-and-by-instance__u500-i10-1hz-1h-every60s.json" + }, + { + "id": "mixed-control-topk10-and-topk50__u5000-i1-1hz-1h-every60s", + "file": "sketch-heavy/mixed-control-topk10-and-topk50__u5000-i1-1hz-1h-every60s.json" + }, + { + "id": "mixed-l2-and-count__r1-10hz-24h-every60s", + "file": "sketch-heavy/mixed-l2-and-count__r1-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-and-distinct__r1-10hz-24h-every60s", + "file": "sketch-heavy/mixed-l2-and-distinct__r1-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-24h-and-15m__r1-10hz-every60s", + "file": "sketch-heavy/mixed-l2-24h-and-15m__r1-10hz-every60s.json" + }, + { + "id": "mixed-repeat-rate-and-l2__r1-10hz-24h-every60s", + "file": "sketch-heavy/mixed-repeat-rate-and-l2__r1-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-and-count__r4-10hz-24h-every60s", + "file": "sketch-heavy/mixed-l2-and-count__r4-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-and-distinct__r4-10hz-24h-every60s", + "file": "sketch-heavy/mixed-l2-and-distinct__r4-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-24h-and-15m__r4-10hz-every60s", + "file": "sketch-heavy/mixed-l2-24h-and-15m__r4-10hz-every60s.json" + }, + { + "id": "mixed-repeat-rate-and-l2__r4-10hz-24h-every60s", + "file": "sketch-heavy/mixed-repeat-rate-and-l2__r4-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-and-count__r8-10hz-24h-every60s", + "file": "sketch-heavy/mixed-l2-and-count__r8-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-and-distinct__r8-10hz-24h-every60s", + "file": "sketch-heavy/mixed-l2-and-distinct__r8-10hz-24h-every60s.json" + }, + { + "id": "mixed-l2-24h-and-15m__r8-10hz-every60s", + "file": "sketch-heavy/mixed-l2-24h-and-15m__r8-10hz-every60s.json" + }, + { + "id": "mixed-repeat-rate-and-l2__r8-10hz-24h-every60s", + "file": "sketch-heavy/mixed-repeat-rate-and-l2__r8-10hz-24h-every60s.json" + }, + { + "id": "mixed-edge-dashboard__u1000-i1-1hz-r4-10hz", + "file": "sketch-heavy/mixed-edge-dashboard__u1000-i1-1hz-r4-10hz.json" + }, + { + "id": "blocked-sql-topk-count-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/blocked-sql-topk-count-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "blocked-sql-topk-bytes-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05", + "file": "sketch-heavy/blocked-sql-topk-bytes-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json" + }, + { + "id": "blocked-sql-l2-client-ip__100000rps-3600s-every60s-releps0.1-d0.1", + "file": "sketch-heavy/blocked-sql-l2-client-ip__100000rps-3600s-every60s-releps0.1-d0.1.json" + }, + { + "id": "blocked-sql-entropy-client-ip__100000rps-3600s-every60s-abseps0.1-d0.1", + "file": "sketch-heavy/blocked-sql-entropy-client-ip__100000rps-3600s-every60s-abseps0.1-d0.1.json" + }, + { + "id": "blocked-promql-entropy__r4-10hz-24h-every60s-abseps0.1-d0.1", + "file": "sketch-heavy/blocked-promql-entropy__r4-10hz-24h-every60s-abseps0.1-d0.1.json" + }, + { + "id": "blocked-sql-distinct-users-no-delta-hll-shape__2000000rps-86400s-every60s-releps0.02", + "file": "sketch-heavy/blocked-sql-distinct-users-no-delta-hll-shape__2000000rps-86400s-every60s-releps0.02.json" + }, + { + "id": "blocked-sql-point-count-one-url__100000rps-3600s-every60s-l1eps0.001-d0.05", + "file": "sketch-heavy/blocked-sql-point-count-one-url__100000rps-3600s-every60s-l1eps0.001-d0.05.json" + } + ] } ] } diff --git a/tools/dag-viewer/workloads/sketch-heavy/blocked-promql-entropy__r4-10hz-24h-every60s-abseps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/blocked-promql-entropy__r4-10hz-24h-every60s-abseps0.1-d0.1.json new file mode 100644 index 000000000..0dca8411d --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/blocked-promql-entropy__r4-10hz-24h-every60s-abseps0.1-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "entropy_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-distinct-users-no-delta-hll-shape__2000000rps-86400s-every60s-releps0.02.json b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-distinct-users-no-delta-hll-shape__2000000rps-86400s-every60s-releps0.02.json new file mode 100644 index 000000000..3852e8bb8 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-distinct-users-no-delta-hll-shape__2000000rps-86400s-every60s-releps0.02.json @@ -0,0 +1,50 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '86400 seconds'", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.02 + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-entropy-client-ip__100000rps-3600s-every60s-abseps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-entropy-client-ip__100000rps-3600s-every60s-abseps0.1-d0.1.json new file mode 100644 index 000000000..9023b24b6 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-entropy-client-ip__100000rps-3600s-every60s-abseps0.1-d0.1.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT -SUM(p * LN(p)) FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY client_ip)", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-l2-client-ip__100000rps-3600s-every60s-releps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-l2-client-ip__100000rps-3600s-every60s-releps0.1-d0.1.json new file mode 100644 index 000000000..bf41bf317 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-l2-client-ip__100000rps-3600s-every60s-releps0.1-d0.1.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (SELECT client_ip, COUNT(*) AS c FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY client_ip)", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-point-count-one-url__100000rps-3600s-every60s-l1eps0.001-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-point-count-one-url__100000rps-3600s-every60s-l1eps0.001-d0.05.json new file mode 100644 index 000000000..cb5b65e2e --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-point-count-one-url__100000rps-3600s-every60s-l1eps0.001-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT COUNT(*) FROM clicks WHERE url = 'url-00042' AND ts >= now() - INTERVAL '3600 seconds'", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.001, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-topk-bytes-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-topk-bytes-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..4dcd63b30 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-topk-bytes-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,53 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT url, SUM(bytes) AS b FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY url ORDER BY b DESC LIMIT 10", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + }, + "series_count": 10000 + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-topk-count-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-topk-count-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..1539c6522 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/blocked-sql-topk-count-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT url, COUNT(*) AS c FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY url ORDER BY c DESC LIMIT 10", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r1-10hz-24h-every60s-releps0.05-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r1-10hz-24h-every60s-releps0.05-d0.1.json new file mode 100644 index 000000000..0a4011728 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r1-10hz-24h-every60s-releps0.05-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "distinct_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r4-10hz-6h-every60s-releps0.05-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r4-10hz-6h-every60s-releps0.05-d0.1.json new file mode 100644 index 000000000..8fae24d08 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r4-10hz-6h-every60s-releps0.05-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "distinct_over_time(dns_query_client[6h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r8-1hz-24h-every300s-releps0.02-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r8-1hz-24h-every300s-releps0.02-d0.1.json new file mode 100644 index 000000000..5aecaad99 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-promql-distinct-clients__r8-1hz-24h-every300s-releps0.02-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "distinct_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.02, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 8.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 1000, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__100000rps-3600s-every60s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__100000rps-3600s-every60s-releps0.05-d0.05.json new file mode 100644 index 000000000..8bf18502c --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__100000rps-3600s-every60s-releps0.05-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT region, COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY region", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-3600s-every60s-releps0.02-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-3600s-every60s-releps0.02-d0.05.json new file mode 100644 index 000000000..afae38443 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-3600s-every60s-releps0.02-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT region, COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY region", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.02, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-86400s-every300s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-86400s-every300s-releps0.05-d0.05.json new file mode 100644 index 000000000..66260ff18 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-region__2000000rps-86400s-every300s-releps0.05-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT region, COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '86400 seconds' GROUP BY region", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-service__100000rps-86400s-every300s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-service__100000rps-86400s-every300s-releps0.05-d0.05.json new file mode 100644 index 000000000..47e7e27ca --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-service__100000rps-86400s-every300s-releps0.05-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT service, COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '86400 seconds' GROUP BY service", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-service__2000000rps-3600s-every60s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-service__2000000rps-3600s-every60s-releps0.05-d0.05.json new file mode 100644 index 000000000..b7e0e1724 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users-by-service__2000000rps-3600s-every60s-releps0.05-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT service, COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY service", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__100000rps-3600s-every60s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__100000rps-3600s-every60s-releps0.05-d0.05.json new file mode 100644 index 000000000..3cbc67874 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__100000rps-3600s-every60s-releps0.05-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds'", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__100000rps-86400s-every300s-releps0.02-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__100000rps-86400s-every300s-releps0.02-d0.05.json new file mode 100644 index 000000000..cd1839376 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__100000rps-86400s-every300s-releps0.02-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '86400 seconds'", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.02, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__2000000rps-3600s-every10s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__2000000rps-3600s-every10s-releps0.05-d0.05.json new file mode 100644 index 000000000..46c3474ef --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__2000000rps-3600s-every10s-releps0.05-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds'", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__2000000rps-86400s-every60s-releps0.02-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__2000000rps-86400s-every60s-releps0.02-d0.05.json new file mode 100644 index 000000000..a53b88521 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/distinct-sql-distinct-users__2000000rps-86400s-every60s-releps0.02-d0.05.json @@ -0,0 +1,52 @@ +{ + "query_workload": { + "language": { + "sql": "datafusion_sql" + }, + "repeating_queries": [ + { + "query": "SELECT COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '86400 seconds'", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.02, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000000.0 + } + }, + "sources": { + "clicks": { + "column_signs": { + "bytes": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000000, + "user_id": 5000000, + "client_ip": 2000000, + "region": 8, + "service": 40 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-control-l2-16-resolvers__r16-10hz-24h-every60s-releps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-control-l2-16-resolvers__r16-10hz-24h-every60s-releps0.1-d0.1.json new file mode 100644 index 000000000..2f94c3c6b --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-control-l2-16-resolvers__r16-10hz-24h-every60s-releps0.1-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 160.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 16, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 16 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-10hz-24h-every60s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-10hz-24h-every60s-releps0.05-d0.05.json new file mode 100644 index 000000000..a56814a4b --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-10hz-24h-every60s-releps0.05-d0.05.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-10hz-6h-every10s-releps0.2-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-10hz-6h-every10s-releps0.2-d0.05.json new file mode 100644 index 000000000..9486e655c --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-10hz-6h-every10s-releps0.2-d0.05.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[6h])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.2, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-1hz-1h-every60s-releps0.1-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-1hz-1h-every60s-releps0.1-d0.05.json new file mode 100644 index 000000000..9889684b5 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r1-1hz-1h-every60s-releps0.1-d0.05.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[1h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 1000, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r2-10hz-24h-every300s-releps0.1-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r2-10hz-24h-every300s-releps0.1-d0.05.json new file mode 100644 index 000000000..d05051051 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r2-10hz-24h-every300s-releps0.1-d0.05.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 20.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 2, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 2 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r2-1hz-5m-every60s-releps0.1-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r2-1hz-5m-every60s-releps0.1-d0.05.json new file mode 100644 index 000000000..18c32e917 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r2-1hz-5m-every60s-releps0.1-d0.05.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[5m])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 2, + "metric_type": "gauge", + "sample_interval": 1000, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 2 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-24h-every300s-releps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-24h-every300s-releps0.1-d0.1.json new file mode 100644 index 000000000..1a2e2fc8d --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-24h-every300s-releps0.1-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-6h-every60s-releps0.05-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-6h-every60s-releps0.05-d0.05.json new file mode 100644 index 000000000..7235d677f --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-10hz-6h-every60s-releps0.05-d0.05.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[6h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-1hz-1h-every10s-releps0.1-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-1hz-1h-every10s-releps0.1-d0.05.json new file mode 100644 index 000000000..dbb702209 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r4-1hz-1h-every10s-releps0.1-d0.05.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[1h])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 4.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 1000, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-1h-every60s-releps0.05-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-1h-every60s-releps0.05-d0.1.json new file mode 100644 index 000000000..be12ed6cd --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-1h-every60s-releps0.05-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[1h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 80.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-24h-every10s-releps0.2-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-24h-every10s-releps0.2-d0.1.json new file mode 100644 index 000000000..dbf306718 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-24h-every10s-releps0.2-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.2, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 80.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-5m-every10s-releps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-5m-every10s-releps0.1-d0.1.json new file mode 100644 index 000000000..d6d143a18 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-10hz-5m-every10s-releps0.1-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[5m])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 80.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-1hz-24h-every60s-releps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-1hz-24h-every60s-releps0.1-d0.1.json new file mode 100644 index 000000000..2a6c95756 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/frequency-moments-univmon-l2__r8-1hz-24h-every60s-releps0.1-d0.1.json @@ -0,0 +1,49 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 8.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 1000, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/gen.py b/tools/dag-viewer/workloads/sketch-heavy/gen.py new file mode 100644 index 000000000..536e4b51c --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/gen.py @@ -0,0 +1,229 @@ +"""Generate the sketch-heavy workload set: the PlanningWorkload files in this +directory, its tables.json, and its entry in ../index.json. + +Each workload targets one sketch family the planner can select (UnivMon for +a frequency L2, Count-Min or Count Sketch with a heap for top-k, KMV for +distinct counts), sits next to one, or is a shape that should plan a sketch +but does not yet (`blocked-*`). File names are +-__, for example +heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05: +top-10 URLs by request rate over 1,000 URLs with one instance each, sampled +at 1 Hz, a 5 min window, evaluated every 60 s, at an L1-normalized error of +0.01 with failure probability 0.05. All metric, table and label names are +synthetic. + +Run from anywhere: python3 tools/dag-viewer/workloads/sketch-heavy/gen.py +""" +import json, os + +HERE = os.path.dirname(os.path.abspath(__file__)) +INDEX = os.path.join(HERE, "..", "index.json") +SET_ID = os.path.basename(HERE) +SQL = {"sql": "datafusion_sql"} +CLICKS = {"name": "clicks", "time_index": 0, "columns": [ + {"name": "ts", "type": "timestamp", "nullable": False}, + {"name": "url", "type": "utf8", "nullable": False}, + {"name": "user_id", "type": "utf8", "nullable": False}, + {"name": "client_ip", "type": "utf8", "nullable": False}, + {"name": "region", "type": "utf8", "nullable": False}, + {"name": "service", "type": "utf8", "nullable": False}, + {"name": "bytes", "type": "float64", "nullable": False}]} + +def target(metric, eps, delta): + return {"accuracy_metric": metric, "max_error": eps, "failure_probability": delta, "accuracy_scope": "whole_result"} + +def q(text, every_s, acc=None): + r = {"query": text, "demand": {"fixed_interval_at": {"interval": every_s * 1000, "evaluation_phase": 0}}, + "predictability": {"predictable": {"known_at": None}}} + r["requirements"] = {"accuracy": acc} if acc else {} + return r + +def wl(lang, queries, sources, rows_per_second): + return {"query_workload": {"language": lang, "repeating_queries": queries}, + "data_workload": {"arrival": {"ingesting": {"rows_per_second": float(rows_per_second)}}, "sources": sources}} + +def http(urls, instances, hz): + """Request counter per (url, instance): one series each.""" + return {"http_requests_total": {"series_count": urls * instances, "metric_type": "counter", + "sample_interval": 1000 // hz, "distinct_values_per_label": {"url": urls, "instance": instances}}} + +def http_bytes(urls, instances, hz): + return {"http_response_bytes": {"series_count": urls * instances, "metric_type": "gauge", + "sample_interval": 1000 // hz, "column_signs": {"value": "non_negative"}, + "distinct_values_per_label": {"url": urls, "instance": instances}}} + +def dns(resolvers, hz): + """One series per resolver; each sample's value is the numeric id of the + client that sent one DNS query (an item stream, the ASAP value model).""" + return {"dns_query_client": {"series_count": resolvers, "metric_type": "gauge", "sample_interval": 1000 // hz, + "column_signs": {"value": "non_negative"}, "distinct_values_per_label": {"resolver": resolvers}}} + +def clicks(**extra): + s = {"column_signs": {"bytes": "non_negative"}, + "distinct_values_per_label": {"url": 1000000, "user_id": 5000000, "client_ip": 2000000, + "region": 8, "service": 40}} + s.update(extra) + return {"clicks": s} + +W = {} # set -> {name: workload} +def add(s, name, w): + W.setdefault(s, {})[name] = w + +# ---------------------------------------------------------------- heavy hitters +TOPK = "topk({k}, sum by (url) (rate(http_requests_total[{w}])))" +for urls, inst, hz, win, every, eps, delta in [ + (1000, 1, 1, "5m", 10, 0.01, 0.05), (1000, 1, 1, "5m", 60, 0.01, 0.05), (1000, 1, 10, "1h", 60, 0.01, 0.05), + (1000, 1, 10, "24h", 300, 0.002, 0.01), (2500, 1, 1, "1h", 60, 0.01, 0.05), (2500, 1, 10, "5m", 10, 0.002, 0.05), + (5000, 1, 1, "5m", 60, 0.01, 0.05), (5000, 1, 10, "1h", 10, 0.01, 0.05), (5000, 1, 1, "24h", 300, 0.002, 0.05), + (1000, 2, 1, "1h", 60, 0.01, 0.05), (2500, 2, 10, "5m", 10, 0.01, 0.05), (2500, 2, 1, "24h", 60, 0.002, 0.05)]: + add("heavy-hitters", f"cmsheap-topk10__u{urls}-i{inst}-{hz}hz-{win}-every{every}s-l1eps{eps}-d{delta}", + wl("promql", [q(TOPK.format(k=10, w=win), every, target("absolute_stream_l1", eps, delta))], + http(urls, inst, hz), urls * inst * hz)) +for urls, hz, win, every, eps, delta in [ + (100, 1, "5m", 60, 0.1, 0.1), (100, 10, "5m", 10, 0.05, 0.1), (100, 1, "1h", 60, 0.02, 0.1), + (300, 1, "5m", 60, 0.05, 0.1), (300, 10, "1h", 60, 0.1, 0.1), (1000, 1, "5m", 60, 0.05, 0.1)]: + add("heavy-hitters", f"csheap-topk10__u{urls}-i1-{hz}hz-{win}-every{every}s-l2eps{eps}-d{delta}", + wl("promql", [q(TOPK.format(k=10, w=win), every, target("absolute_stream_l2", eps, delta))], + http(urls, 1, hz), urls * hz)) +# Controls: exact wins or the target cannot be certified. +add("heavy-hitters", "control-topk10-10-instances-per-url__u1000-i10-10hz-1h-every60s-l1eps0.01-d0.05", + wl("promql", [q(TOPK.format(k=10, w="1h"), 60, target("absolute_stream_l1", 0.01, 0.05))], http(1000, 10, 10), 100000)) +add("heavy-hitters", "control-topk10-sum-over-time__u1000-i1-1hz-1h-every60s-l1eps0.01-d0.05", + wl("promql", [q("topk(10, sum by (url) (sum_over_time(http_response_bytes[1h])))", 60, target("absolute_stream_l1", 0.01, 0.05))], + http_bytes(1000, 1, 1), 1000)) +add("heavy-hitters", "control-topk10-delta-too-small__u5000-i1-1hz-1h-every60s-l1eps0.01-d0.01", + wl("promql", [q(TOPK.format(k=10, w="1h"), 60, target("absolute_stream_l1", 0.01, 0.01))], http(5000, 1, 1), 5000)) +add("heavy-hitters", "control-topk10-10000-urls__u10000-i1-1hz-1h-every60s-l1eps0.01-d0.05", + wl("promql", [q(TOPK.format(k=10, w="1h"), 60, target("absolute_stream_l1", 0.01, 0.05))], http(10000, 1, 1), 10000)) + +# ---------------------------------------------------------- frequency moments +L2 = "l2_over_time(dns_query_client[{w}])" +for res, hz, win, every, eps, delta in [ + (1, 1, "1h", 60, 0.1, 0.05), (1, 10, "24h", 60, 0.05, 0.05), (1, 10, "6h", 10, 0.2, 0.05), + (2, 10, "24h", 300, 0.1, 0.05), (4, 1, "1h", 10, 0.1, 0.05), (4, 10, "6h", 60, 0.05, 0.05), + (4, 10, "24h", 300, 0.1, 0.1), (8, 1, "24h", 60, 0.1, 0.1), (8, 10, "24h", 10, 0.2, 0.1), + (8, 10, "1h", 60, 0.05, 0.1), (2, 1, "5m", 60, 0.1, 0.05), (8, 10, "5m", 10, 0.1, 0.1)]: + add("frequency-moments", f"univmon-l2__r{res}-{hz}hz-{win}-every{every}s-releps{eps}-d{delta}", + wl("promql", [q(L2.format(w=win), every, target("relative", eps, delta))], dns(res, hz), res * hz)) +add("frequency-moments", "control-l2-16-resolvers__r16-10hz-24h-every60s-releps0.1-d0.1", + wl("promql", [q(L2.format(w="24h"), 60, target("relative", 0.1, 0.1))], dns(16, 10), 160)) + +# ------------------------------------------------------------------ distinct +DQ_ALL = "SELECT COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '{s} seconds'" +DQ_GRP = "SELECT {g}, COUNT(DISTINCT user_id) FROM clicks WHERE ts >= now() - INTERVAL '{s} seconds' GROUP BY {g}" +for grp, rps, secs, every, eps in [ + (None, 100000, 3600, 60, 0.05), (None, 100000, 86400, 300, 0.02), (None, 2000000, 3600, 10, 0.05), + (None, 2000000, 86400, 60, 0.02), ("region", 100000, 3600, 60, 0.05), ("region", 2000000, 86400, 300, 0.05), + ("region", 2000000, 3600, 60, 0.02), ("service", 100000, 86400, 300, 0.05), ("service", 2000000, 3600, 60, 0.05)]: + text = DQ_ALL.format(s=secs) if grp is None else DQ_GRP.format(g=grp, s=secs) + add("distinct", f"sql-distinct-users{'-by-' + grp if grp else ''}__{rps}rps-{secs}s-every{every}s-releps{eps}-d0.05", + wl(SQL, [q(text, every, target("relative", eps, 0.05))], clicks(), rps)) +for res, hz, win, every, eps in [(1, 10, "24h", 60, 0.05), (4, 10, "6h", 60, 0.05), (8, 1, "24h", 300, 0.02)]: + add("distinct", f"promql-distinct-clients__r{res}-{hz}hz-{win}-every{every}s-releps{eps}-d0.1", + wl("promql", [q(f"distinct_over_time(dns_query_client[{win}])", every, target("relative", eps, 0.1))], dns(res, hz), res * hz)) + +# --------------------------------------------------------------------- mixed +X1h = "sum by (url) (rate(http_requests_total[1h]))" +for urls in [1000, 5000]: + add("mixed", f"topk-and-total__u{urls}-i1-1hz-1h-every60s", + wl("promql", [q(f"topk(10, {X1h})", 60, target("absolute_stream_l1", 0.01, 0.05)), + q("sum(rate(http_requests_total[1h]))", 60)], http(urls, 1, 1), urls)) + add("mixed", f"topk-dashboard-and-alert__u{urls}-i1-1hz-1h-every10s-and-60s", + wl("promql", [q(f"topk(10, {X1h})", 10, target("absolute_stream_l1", 0.01, 0.05)), + q(f"topk(10, {X1h})", 60, target("absolute_stream_l1", 0.002, 0.05))], http(urls, 1, 1), urls)) + add("mixed", f"topk-by-url-and-by-instance__u{urls // 10}-i10-1hz-1h-every60s", + wl("promql", [q(f"topk(10, {X1h})", 60, target("absolute_stream_l1", 0.01, 0.05)), + q("topk(10, sum by (instance) (rate(http_requests_total[1h])))", 60, target("absolute_stream_l1", 0.01, 0.05))], + http(urls // 10, 10, 1), urls)) + add("mixed", f"control-topk10-and-topk50__u{urls}-i1-1hz-1h-every60s", + wl("promql", [q(f"topk(10, {X1h})", 60, target("absolute_stream_l1", 0.01, 0.05)), + q(f"topk(50, {X1h})", 60, target("absolute_stream_l1", 0.01, 0.05))], http(urls, 1, 1), urls)) +for res in [1, 4, 8]: + R = target("relative", 0.1, 0.1) + add("mixed", f"l2-and-count__r{res}-10hz-24h-every60s", + wl("promql", [q(L2.format(w="24h"), 60, R), q("count_over_time(dns_query_client[24h])", 60)], dns(res, 10), res * 10)) + add("mixed", f"l2-and-distinct__r{res}-10hz-24h-every60s", + wl("promql", [q(L2.format(w="24h"), 60, R), q("distinct_over_time(dns_query_client[24h])", 60, target("relative", 0.05, 0.1))], + dns(res, 10), res * 10)) + add("mixed", f"l2-24h-and-15m__r{res}-10hz-every60s", + wl("promql", [q(L2.format(w="24h"), 60, R), q(L2.format(w="15m"), 60, R)], dns(res, 10), res * 10)) + add("mixed", f"repeat-rate-and-l2__r{res}-10hz-24h-every60s", + wl("promql", [q("l2_over_time(dns_query_client[24h]) / count_over_time(dns_query_client[24h])", 60, R), + q(L2.format(w="24h"), 60, R)], dns(res, 10), res * 10)) +src = {} +src.update(http(1000, 1, 1)); src.update(dns(4, 10)) +add("mixed", "edge-dashboard__u1000-i1-1hz-r4-10hz", + wl("promql", [q(f"topk(10, {X1h})", 60, target("absolute_stream_l1", 0.01, 0.05)), + q("sum(rate(http_requests_total[5m]))", 10), + q(L2.format(w="24h"), 60, target("relative", 0.1, 0.1)), + q("distinct_over_time(dns_query_client[24h])", 300, target("relative", 0.02, 0.1))], src, 1040)) + +# ------------------------------------------- blocked (shapes no sketch serves yet) +L1T = target("absolute_stream_l1", 0.01, 0.05) +add("blocked", "sql-topk-count-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05", + wl(SQL, [q("SELECT url, COUNT(*) AS c FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY url ORDER BY c DESC LIMIT 10", 60, L1T)], + clicks(distinct_values_per_label={"url": 1000, "user_id": 5000000, "client_ip": 2000000, "region": 8, "service": 40}), 100000)) +add("blocked", "sql-topk-bytes-urls__1000urls-100000rps-3600s-every60s-l1eps0.01-d0.05", + wl(SQL, [q("SELECT url, SUM(bytes) AS b FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY url ORDER BY b DESC LIMIT 10", 60, L1T)], + clicks(series_count=10000, distinct_values_per_label={"url": 1000, "user_id": 5000000, "client_ip": 2000000, "region": 8, "service": 40}), 100000)) +add("blocked", "sql-l2-client-ip__100000rps-3600s-every60s-releps0.1-d0.1", + wl(SQL, [q("SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (SELECT client_ip, COUNT(*) AS c FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY client_ip)", 60, target("relative", 0.1, 0.1))], + clicks(), 100000)) +add("blocked", "sql-entropy-client-ip__100000rps-3600s-every60s-abseps0.1-d0.1", + wl(SQL, [q("SELECT -SUM(p * LN(p)) FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM clicks WHERE ts >= now() - INTERVAL '3600 seconds' GROUP BY client_ip)", 60, target("absolute", 0.1, 0.1))], + clicks(), 100000)) +add("blocked", "promql-entropy__r4-10hz-24h-every60s-abseps0.1-d0.1", + wl("promql", [q("entropy_over_time(dns_query_client[24h])", 60, target("absolute", 0.1, 0.1))], dns(4, 10), 40)) +add("blocked", "sql-distinct-users-no-delta-hll-shape__2000000rps-86400s-every60s-releps0.02", + wl(SQL, [q(DQ_ALL.format(s=86400), 60, {"accuracy_metric": "relative", "max_error": 0.02})], clicks(), 2000000)) +add("blocked", "sql-point-count-one-url__100000rps-3600s-every60s-l1eps0.001-d0.05", + wl(SQL, [q("SELECT COUNT(*) FROM clicks WHERE url = 'url-00042' AND ts >= now() - INTERVAL '3600 seconds'", 60, target("absolute_stream_l1", 0.001, 0.05))], + clicks(), 100000)) + + +SET = { + "id": SET_ID, + "name": "Sketch-heavy", + "description": ( + "Workloads where a sketch family is the cheapest plan that meets the accuracy target, with controls " + "next to them. heavy-hitters: topk(10, sum by (url) (rate(http_requests_total[W]))) over 100 to 10,000 " + "URLs at L1- or L2-normalized targets (Count-Min or Count Sketch with a heap). frequency-moments: " + "l2_over_time over 1 to 16 value streams (UnivMon). distinct: SQL COUNT(DISTINCT user_id) at 100,000 " + "or 2,000,000 rows/s and PromQL distinct_over_time (KMV). mixed: several queries that could share one " + "sketch. blocked: SQL top-k, L2, entropy, point counts and HLL-shaped distinct counts, which no sketch " + "serves yet."), + "source": ( + "Synthetic, written by gen.py in this directory (metric, table and label names are made up). Each " + "workload declares only the data facts the shape implies: series count, sample interval, metric type, " + "distinct values per grouping label and column signs; no stream norms. SQL workloads read the clicks " + "table in tables.json. Planned with the default deployment (raw data retained) and cost model, " + "accuracy guarantees enforced."), + "accuracy_guarantees": "enforced", +} + + +def file_name(subset, name): + return f"{subset}-{name}.json" + + +if __name__ == "__main__": + for old in os.listdir(HERE): + if old.endswith(".json"): + os.remove(os.path.join(HERE, old)) + with open(os.path.join(HERE, "tables.json"), "w") as f: + json.dump([CLICKS], f, indent=1) + f.write("\n") + entries = [] + for subset, workloads in W.items(): + for name, workload in workloads.items(): + with open(os.path.join(HERE, file_name(subset, name)), "w") as f: + json.dump(workload, f, indent=1) + f.write("\n") + entries.append({"id": f"{subset}-{name}", "file": f"{SET_ID}/{file_name(subset, name)}"}) + with open(INDEX) as f: + index = json.load(f) + index["sets"] = [s for s in index["sets"] if s["id"] != SET_ID] + [dict(SET, workloads=entries)] + with open(INDEX, "w") as f: + json.dump(index, f, indent=2, ensure_ascii=False) + f.write("\n") + print(len(entries), "workloads in", {s: len(v) for s, v in W.items()}) diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-1h-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-1h-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..2a6678160 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-1h-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-24h-every300s-l1eps0.002-d0.01.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-24h-every300s-l1eps0.002-d0.01.json new file mode 100644 index 000000000..8256f9e0d --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-10hz-24h-every300s-l1eps0.002-d0.01.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[24h])))", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.002, + "failure_probability": 0.01, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every10s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every10s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..45e49ce5e --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every10s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..a3541fe87 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i2-1hz-1h-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i2-1hz-1h-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..61d99b200 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u1000-i2-1hz-1h-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 2000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 2 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-10hz-5m-every10s-l1eps0.002-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-10hz-5m-every10s-l1eps0.002-d0.05.json new file mode 100644 index 000000000..9062b5e45 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-10hz-5m-every10s-l1eps0.002-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.002, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 25000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 2500, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 2500, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-1hz-1h-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-1hz-1h-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..285161a0d --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i1-1hz-1h-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2500.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 2500, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 2500, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-10hz-5m-every10s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-10hz-5m-every10s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..9b48645b3 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-10hz-5m-every10s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 50000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 2500, + "instance": 2 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-1hz-24h-every60s-l1eps0.002-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-1hz-24h-every60s-l1eps0.002-d0.05.json new file mode 100644 index 000000000..303f0fe74 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u2500-i2-1hz-24h-every60s-l1eps0.002-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[24h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.002, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 2500, + "instance": 2 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-10hz-1h-every10s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-10hz-1h-every10s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..faf84ed20 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-10hz-1h-every10s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 50000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 5000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-24h-every300s-l1eps0.002-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-24h-every300s-l1eps0.002-d0.05.json new file mode 100644 index 000000000..d45a30897 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-24h-every300s-l1eps0.002-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[24h])))", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.002, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 5000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..dd51141d7 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-cmsheap-topk10__u5000-i1-1hz-5m-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 5000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-10-instances-per-url__u1000-i10-10hz-1h-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-10-instances-per-url__u1000-i10-10hz-1h-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..52bd19733 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-10-instances-per-url__u1000-i10-10hz-1h-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 1000, + "instance": 10 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-10000-urls__u10000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-10000-urls__u10000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..1a1119261 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-10000-urls__u10000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 10000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-delta-too-small__u5000-i1-1hz-1h-every60s-l1eps0.01-d0.01.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-delta-too-small__u5000-i1-1hz-1h-every60s-l1eps0.01-d0.01.json new file mode 100644 index 000000000..beb69c64d --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-delta-too-small__u5000-i1-1hz-1h-every60s-l1eps0.01-d0.01.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.01, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 5000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-sum-over-time__u1000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-sum-over-time__u1000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json new file mode 100644 index 000000000..524c89b76 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-control-topk10-sum-over-time__u1000-i1-1hz-1h-every60s-l1eps0.01-d0.05.json @@ -0,0 +1,50 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (sum_over_time(http_response_bytes[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_response_bytes": { + "series_count": 1000, + "metric_type": "gauge", + "sample_interval": 1000, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-10hz-5m-every10s-l2eps0.05-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-10hz-5m-every10s-l2eps0.05-d0.1.json new file mode 100644 index 000000000..2f76370b7 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-10hz-5m-every10s-l2eps0.05-d0.1.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l2", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 100, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 100, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-1h-every60s-l2eps0.02-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-1h-every60s-l2eps0.02-d0.1.json new file mode 100644 index 000000000..8998e4503 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-1h-every60s-l2eps0.02-d0.1.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l2", + "max_error": 0.02, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 100, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 100, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-5m-every60s-l2eps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-5m-every60s-l2eps0.1-d0.1.json new file mode 100644 index 000000000..50bd04712 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u100-i1-1hz-5m-every60s-l2eps0.1-d0.1.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l2", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 100.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 100, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 100, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u1000-i1-1hz-5m-every60s-l2eps0.05-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u1000-i1-1hz-5m-every60s-l2eps0.05-d0.1.json new file mode 100644 index 000000000..887c87a30 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u1000-i1-1hz-5m-every60s-l2eps0.05-d0.1.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l2", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-10hz-1h-every60s-l2eps0.1-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-10hz-1h-every60s-l2eps0.1-d0.1.json new file mode 100644 index 000000000..1f3220e64 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-10hz-1h-every60s-l2eps0.1-d0.1.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l2", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 3000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 300, + "metric_type": "counter", + "sample_interval": 100, + "distinct_values_per_label": { + "url": 300, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-1hz-5m-every60s-l2eps0.05-d0.1.json b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-1hz-5m-every60s-l2eps0.05-d0.1.json new file mode 100644 index 000000000..d73350ddc --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/heavy-hitters-csheap-topk10__u300-i1-1hz-5m-every60s-l2eps0.05-d0.1.json @@ -0,0 +1,47 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l2", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 300.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 300, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 300, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-control-topk10-and-topk50__u1000-i1-1hz-1h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-control-topk10-and-topk50__u1000-i1-1hz-1h-every60s.json new file mode 100644 index 000000000..8dea37def --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-control-topk10-and-topk50__u1000-i1-1hz-1h-every60s.json @@ -0,0 +1,69 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "topk(50, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-control-topk10-and-topk50__u5000-i1-1hz-1h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-control-topk10-and-topk50__u5000-i1-1hz-1h-every60s.json new file mode 100644 index 000000000..4591fc0af --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-control-topk10-and-topk50__u5000-i1-1hz-1h-every60s.json @@ -0,0 +1,69 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "topk(50, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 5000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-edge-dashboard__u1000-i1-1hz-r4-10hz.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-edge-dashboard__u1000-i1-1hz-r4-10hz.json new file mode 100644 index 000000000..c394d7ac0 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-edge-dashboard__u1000-i1-1hz-r4-10hz.json @@ -0,0 +1,117 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "sum(rate(http_requests_total[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": {} + }, + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "distinct_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 300000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.02, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1040.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + }, + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r1-10hz-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r1-10hz-every60s.json new file mode 100644 index 000000000..82686973e --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r1-10hz-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "l2_over_time(dns_query_client[15m])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r4-10hz-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r4-10hz-every60s.json new file mode 100644 index 000000000..1616c209d --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r4-10hz-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "l2_over_time(dns_query_client[15m])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r8-10hz-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r8-10hz-every60s.json new file mode 100644 index 000000000..6922d0001 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-24h-and-15m__r8-10hz-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "l2_over_time(dns_query_client[15m])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 80.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r1-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r1-10hz-24h-every60s.json new file mode 100644 index 000000000..3fa8c81c0 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r1-10hz-24h-every60s.json @@ -0,0 +1,64 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "count_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": {} + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r4-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r4-10hz-24h-every60s.json new file mode 100644 index 000000000..87c4c760f --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r4-10hz-24h-every60s.json @@ -0,0 +1,64 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "count_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": {} + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r8-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r8-10hz-24h-every60s.json new file mode 100644 index 000000000..7e399abd4 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-count__r8-10hz-24h-every60s.json @@ -0,0 +1,64 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "count_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": {} + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 80.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r1-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r1-10hz-24h-every60s.json new file mode 100644 index 000000000..7074ab3ba --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r1-10hz-24h-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "distinct_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r4-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r4-10hz-24h-every60s.json new file mode 100644 index 000000000..4d0cba8d6 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r4-10hz-24h-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "distinct_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r8-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r8-10hz-24h-every60s.json new file mode 100644 index 000000000..ab6d12e5b --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-l2-and-distinct__r8-10hz-24h-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "distinct_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.05, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 80.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r1-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r1-10hz-24h-every60s.json new file mode 100644 index 000000000..da9c2b6ac --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r1-10hz-24h-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h]) / count_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 10.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r4-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r4-10hz-24h-every60s.json new file mode 100644 index 000000000..f4d11aed9 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r4-10hz-24h-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h]) / count_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 40.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 4, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 4 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r8-10hz-24h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r8-10hz-24h-every60s.json new file mode 100644 index 000000000..c227b5abc --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-repeat-rate-and-l2__r8-10hz-24h-every60s.json @@ -0,0 +1,71 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "l2_over_time(dns_query_client[24h]) / count_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "l2_over_time(dns_query_client[24h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "relative", + "max_error": 0.1, + "failure_probability": 0.1, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 80.0 + } + }, + "sources": { + "dns_query_client": { + "series_count": 8, + "metric_type": "gauge", + "sample_interval": 100, + "column_signs": { + "value": "non_negative" + }, + "distinct_values_per_label": { + "resolver": 8 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-and-total__u1000-i1-1hz-1h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-and-total__u1000-i1-1hz-1h-every60s.json new file mode 100644 index 000000000..d6e0834cf --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-and-total__u1000-i1-1hz-1h-every60s.json @@ -0,0 +1,62 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "sum(rate(http_requests_total[1h]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": {} + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-and-total__u5000-i1-1hz-1h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-and-total__u5000-i1-1hz-1h-every60s.json new file mode 100644 index 000000000..cb019137b --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-and-total__u5000-i1-1hz-1h-every60s.json @@ -0,0 +1,62 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "sum(rate(http_requests_total[1h]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": {} + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 5000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-by-url-and-by-instance__u100-i10-1hz-1h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-by-url-and-by-instance__u100-i10-1hz-1h-every60s.json new file mode 100644 index 000000000..a58da598b --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-by-url-and-by-instance__u100-i10-1hz-1h-every60s.json @@ -0,0 +1,69 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "topk(10, sum by (instance) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 100, + "instance": 10 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-by-url-and-by-instance__u500-i10-1hz-1h-every60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-by-url-and-by-instance__u500-i10-1hz-1h-every60s.json new file mode 100644 index 000000000..c8af2ed8f --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-by-url-and-by-instance__u500-i10-1hz-1h-every60s.json @@ -0,0 +1,69 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "topk(10, sum by (instance) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 500, + "instance": 10 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-dashboard-and-alert__u1000-i1-1hz-1h-every10s-and-60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-dashboard-and-alert__u1000-i1-1hz-1h-every10s-and-60s.json new file mode 100644 index 000000000..184ba0c19 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-dashboard-and-alert__u1000-i1-1hz-1h-every10s-and-60s.json @@ -0,0 +1,69 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.002, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 1000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 1000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 1000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-dashboard-and-alert__u5000-i1-1hz-1h-every10s-and-60s.json b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-dashboard-and-alert__u5000-i1-1hz-1h-every10s-and-60s.json new file mode 100644 index 000000000..00248036d --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/mixed-topk-dashboard-and-alert__u5000-i1-1hz-1h-every10s-and-60s.json @@ -0,0 +1,69 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.01, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + }, + { + "query": "topk(10, sum by (url) (rate(http_requests_total[1h])))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "predictability": { + "predictable": { + "known_at": null + } + }, + "requirements": { + "accuracy": { + "accuracy_metric": "absolute_stream_l1", + "max_error": 0.002, + "failure_probability": 0.05, + "accuracy_scope": "whole_result" + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 5000.0 + } + }, + "sources": { + "http_requests_total": { + "series_count": 5000, + "metric_type": "counter", + "sample_interval": 1000, + "distinct_values_per_label": { + "url": 5000, + "instance": 1 + } + } + } + } +} diff --git a/tools/dag-viewer/workloads/sketch-heavy/tables.json b/tools/dag-viewer/workloads/sketch-heavy/tables.json new file mode 100644 index 000000000..6a1410f01 --- /dev/null +++ b/tools/dag-viewer/workloads/sketch-heavy/tables.json @@ -0,0 +1,43 @@ +[ + { + "name": "clicks", + "time_index": 0, + "columns": [ + { + "name": "ts", + "type": "timestamp", + "nullable": false + }, + { + "name": "url", + "type": "utf8", + "nullable": false + }, + { + "name": "user_id", + "type": "utf8", + "nullable": false + }, + { + "name": "client_ip", + "type": "utf8", + "nullable": false + }, + { + "name": "region", + "type": "utf8", + "nullable": false + }, + { + "name": "service", + "type": "utf8", + "nullable": false + }, + { + "name": "bytes", + "type": "float64", + "nullable": false + } + ] + } +]