diff --git a/.diskcheck.txt b/.diskcheck.txt new file mode 100644 index 00000000..50689fdd --- /dev/null +++ b/.diskcheck.txt @@ -0,0 +1,13 @@ +246M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/external-bench +258M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/harness-impl +301M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/r1236 +339M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/dl +378M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/p1m1 +392M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/s2cov +532M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/fix1237 +563M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/r1309fix +711M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/kmvfit +734M /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/dxrefit +1.6G /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/final-review +1.6G /tmp/claude-20002/-mydata/e1ec8edf-ca45-4611-ad77-feb4e4f5b583/scratchpad/trajdb +Tue Nov 17 05:48:05 AM UTC 2026 diff --git a/Cargo.lock b/Cargo.lock index f18c3a77..b598c554 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "ahash" version = "0.8.12" @@ -32,21 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -127,12 +106,6 @@ dependencies = [ "object", ] -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - [[package]] name = "arrow" version = "58.4.0" @@ -357,10 +330,32 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "asap-accuracy" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" +dependencies = [ + "asap-types", + "asap_sketchlib", + "serde_json", +] + +[[package]] +name = "asap-cost" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" +dependencies = [ + "asap-types", + "asap_sketchlib", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "asap-executor" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ "ahash", "arrow", @@ -372,13 +367,12 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.20", - "tracing", ] [[package]] name = "asap-frontend-common" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ "asap-types", "serde", @@ -388,7 +382,7 @@ dependencies = [ [[package]] name = "asap-frontend-metricsql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ "asap-frontend-common", "asap-types", @@ -399,7 +393,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ "asap-frontend-common", "asap-types", @@ -409,7 +403,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ "asap-frontend-common", "asap-sql-function-catalog", @@ -419,68 +413,88 @@ dependencies = [ ] [[package]] -name = "asap-logical-asap-optimization" +name = "asap-planner" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ + "asap-accuracy", + "asap-cost", + "asap-executor", + "asap-frontend-metricsql", + "asap-frontend-promql", + "asap-frontend-sql", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", + "asap-stage1-pass2-reuse-candidates", + "asap-stage2-physical-optimization", + "asap-stage3-plan-selection", "asap-types", - "asap_sketchlib", "serde", "serde_json", "thiserror 2.0.20", ] [[package]] -name = "asap-physical-asap-optimization" +name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" + +[[package]] +name = "asap-stage0-memo" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ - "asap-logical-asap-optimization", "asap-types", - "thiserror 2.0.20", ] [[package]] -name = "asap-plan-selection" +name = "asap-stage1-pass1-per-computation-candidates" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", "asap-types", - "asap_sketchlib", - "serde", - "serde_json", "thiserror 2.0.20", ] [[package]] -name = "asap-planner" +name = "asap-stage1-pass2-reuse-candidates" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ - "asap-executor", - "asap-frontend-metricsql", - "asap-frontend-promql", - "asap-frontend-sql", - "asap-logical-asap-optimization", - "asap-plan-selection", + "asap-accuracy", "asap-types", - "serde", - "serde_json", "thiserror 2.0.20", ] [[package]] -name = "asap-sql-function-catalog" +name = "asap-stage2-physical-optimization" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" +dependencies = [ + "asap-types", +] + +[[package]] +name = "asap-stage3-plan-selection" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" +dependencies = [ + "asap-accuracy", + "asap-cost", + "asap-types", + "good_lp", + "highs", + "serde", + "serde_json", + "thiserror 2.0.20", +] [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ + "asap_sketchlib", "serde", "serde_json", "thiserror 2.0.20", @@ -489,7 +503,7 @@ dependencies = [ [[package]] name = "asap_sketchlib" version = "0.4.0" -source = "git+https://github.com/ProjectASAP/asap_sketchlib?rev=63e565506426020c505ef0b84e0c91f450d09e7c#63e565506426020c505ef0b84e0c91f450d09e7c" +source = "git+https://github.com/ProjectASAP/asap_sketchlib?rev=21b63578c2e612e153f5aaa4d3848f1035a13c60#21b63578c2e612e153f5aaa4d3848f1035a13c60" dependencies = [ "rand 0.9.5", "rmp-serde", @@ -499,18 +513,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "async-compression" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844" -dependencies = [ - "compression-codecs", - "compression-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-trait" version = "0.1.92" @@ -625,79 +627,49 @@ dependencies = [ [[package]] name = "bincode" -version = "1.3.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" dependencies = [ + "bincode_derive", "serde", + "unty", ] [[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" - -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "blake3" -version = "1.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" -dependencies = [ - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-buffer" -version = "0.12.1" +name = "bincode_derive" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" dependencies = [ - "hybrid-array", + "virtue", ] [[package]] -name = "brotli" -version = "8.0.4" +name = "bindgen" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", ] [[package]] -name = "brotli-decompressor" -version = "5.0.3" +name = "bitflags" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bumpalo" @@ -705,27 +677,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "bzip2" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" -dependencies = [ - "libbz2-rs-sys", -] - [[package]] name = "cactus" version = "1.0.7" @@ -741,7 +698,16 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", ] [[package]] @@ -758,16 +724,17 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "cfgrammar" -version = "0.13.10" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" +checksum = "b5ef80386dea3cda50570f22e1c6a474f0cc688ae220fc584fef354acaf41f2b" dependencies = [ + "bincode", "indexmap", - "lazy_static", "num-traits", + "proc-macro2", + "quote", "regex", - "serde", - "vob 4.0.1", + "vob", ] [[package]] @@ -805,6 +772,17 @@ dependencies = [ "phf 0.12.1", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.6.6" @@ -845,6 +823,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -858,36 +845,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] -[[package]] -name = "compression-codecs" -version = "0.4.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df" -dependencies = [ - "bzip2", - "compression-core", - "flate2", - "liblzma", - "memchr", - "zstd", - "zstd-safe", -] - -[[package]] -name = "compression-core" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" - -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - [[package]] name = "const-random" version = "0.1.18" @@ -908,12 +868,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" - [[package]] name = "control_plane" version = "0.1.0" @@ -944,15 +898,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" -dependencies = [ - "cfg-if", -] - [[package]] name = "crossbeam-utils" version = "0.8.23" @@ -965,25 +910,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - [[package]] name = "csv" version = "1.4.0" @@ -1054,7 +980,6 @@ dependencies = [ "arrow", "arrow-schema", "async-trait", - "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1064,7 +989,6 @@ dependencies = [ "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", - "datafusion-datasource-parquet", "datafusion-execution", "datafusion-expr", "datafusion-expr-common", @@ -1081,21 +1005,17 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "datafusion-sql", - "flate2", "futures", "indexmap", - "itertools", - "liblzma", + "itertools 0.14.0", "log", "object_store", "parking_lot", - "parquet", "sqlparser", "tempfile", "tokio", "url", "uuid", - "zstd", ] [[package]] @@ -1116,7 +1036,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -1141,7 +1061,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", ] @@ -1160,11 +1080,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "libc", "log", "object_store", - "parquet", "recursive", "sqlparser", "tokio", @@ -1190,10 +1109,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", - "async-compression", "async-trait", "bytes", - "bzip2", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -1204,19 +1121,15 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", - "flate2", "futures", "glob", - "itertools", - "liblzma", + "itertools 0.14.0", "log", "object_store", "parking_lot", "rand 0.9.5", "tokio", - "tokio-util", "url", - "zstd", ] [[package]] @@ -1238,7 +1151,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.14.0", "object_store", "tokio", ] @@ -1289,37 +1202,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "datafusion-datasource-parquet" -version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", - "futures", - "itertools", - "log", - "object_store", - "parking_lot", - "parquet", - "tokio", -] - [[package]] name = "datafusion-doc" version = "54.1.0" @@ -1365,7 +1247,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", - "itertools", + "itertools 0.14.0", "recursive", "serde_json", "sqlparser", @@ -1380,7 +1262,7 @@ dependencies = [ "arrow", "datafusion-common", "indexmap", - "itertools", + "itertools 0.14.0", ] [[package]] @@ -1392,8 +1274,6 @@ dependencies = [ "arrow", "arrow-buffer", "base64", - "blake2", - "blake3", "chrono", "chrono-tz", "datafusion-common", @@ -1404,14 +1284,12 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools", + "itertools 0.14.0", "log", - "md-5", "memchr", "num-traits", "rand 0.9.5", "regex", - "sha2", "uuid", ] @@ -1467,7 +1345,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools", + "itertools 0.14.0", "itoa", "log", "memchr", @@ -1540,7 +1418,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap", - "itertools", + "itertools 0.14.0", "log", "recursive", "regex", @@ -1562,7 +1440,7 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "parking_lot", "petgraph", "recursive", @@ -1581,7 +1459,7 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools", + "itertools 0.14.0", ] [[package]] @@ -1596,7 +1474,7 @@ dependencies = [ "datafusion-expr-common", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "parking_lot", "pin-project", ] @@ -1616,7 +1494,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools", + "itertools 0.14.0", "recursive", ] @@ -1645,7 +1523,7 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "log", "num-traits", "parking_lot", @@ -1708,28 +1586,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "crypto-common 0.1.7", - "subtle", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "const-oid", - "crypto-common 0.2.2", -] - [[package]] name = "displaydoc" version = "0.2.7" @@ -1769,7 +1625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1810,17 +1666,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "flate2" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" -dependencies = [ - "crc32fast", - "miniz_oxide", - "zlib-rs", -] - [[package]] name = "fnv" version = "1.0.7" @@ -1936,23 +1781,13 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getopts" version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -2000,6 +1835,16 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "good_lp" +version = "1.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4255eb1d65eea322f73a326038f2bcec19dae13ca50fa557da19b03274b4d56" +dependencies = [ + "fnv", + "highs", +] + [[package]] name = "half" version = "2.7.1" @@ -2050,6 +1895,26 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "highs" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfc791427e78fdc1a0a37983f79c20f86a0b7b7dfdb5d6a57a6f72d77cd9074" +dependencies = [ + "highs-sys", + "log", +] + +[[package]] +name = "highs-sys" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4da0c52322648e2360a5a8896d23301fa5ed99ed7607c5faf229bd0bbb30d3f" +dependencies = [ + "bindgen", + "cmake", +] + [[package]] name = "http" version = "1.5.0" @@ -2101,15 +1966,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" -[[package]] -name = "hybrid-array" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" -dependencies = [ - "typenum", -] - [[package]] name = "hyper" version = "1.11.1" @@ -2308,12 +2164,6 @@ dependencies = [ "hashbrown 0.17.1", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.2" @@ -2326,6 +2176,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2425,12 +2284,6 @@ dependencies = [ "lexical-util", ] -[[package]] -name = "libbz2-rs-sys" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" - [[package]] name = "libc" version = "0.2.189" @@ -2438,23 +2291,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] -name = "liblzma" -version = "0.4.8" +name = "libloading" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ - "liblzma-sys", -] - -[[package]] -name = "liblzma-sys" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" -dependencies = [ - "cc", - "libc", - "pkg-config", + "cfg-if", + "windows-link", ] [[package]] @@ -2525,56 +2368,62 @@ dependencies = [ [[package]] name = "lrlex" -version = "0.13.10" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71364e868116ee891b0f93559eb9eca5675bec28b22d33c58481e66c3951d7e" +checksum = "6faf26145c0571bcecf7887802d7dc2947154cb438801d6c90ef3633e68b2365" dependencies = [ + "bincode", "cfgrammar", "getopts", - "lazy_static", + "glob", "lrpar", "num-traits", + "prettyplease", + "proc-macro2", "quote", "regex", "regex-syntax", - "serde", + "syn 2.0.119", "vergen", ] [[package]] name = "lrpar" -version = "0.13.10" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b265a81193d94c92d1c9c715498d6fa505bce3f789ceecb24ab5d6fa2dbc71" +checksum = "402f799a7cfbaf548546e16191a15b7abaeb16ac25bbff5ab41fd93d7548d7a9" dependencies = [ "bincode", "cactus", "cfgrammar", "filetime", "indexmap", - "lazy_static", "lrtable", "num-traits", "packedvec", + "prettyplease", + "proc-macro2", + "quote", "regex", - "serde", - "static_assertions", + "syn 2.0.119", + "unicode-width 0.1.14", "vergen", - "vob 4.0.1", + "vob", + "web-time", ] [[package]] name = "lrtable" -version = "0.13.10" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de" +checksum = "bcbe6a005a604615ab7aab7d1f74be563e508f5c6e42f85fbf65c38fe1f5f268" dependencies = [ + "bincode", "cfgrammar", "fnv", "num-traits", - "serde", "sparsevec", - "vob 4.0.1", + "vob", ] [[package]] @@ -2607,16 +2456,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "md-5" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" -dependencies = [ - "cfg-if", - "digest 0.11.3", -] - [[package]] name = "memchr" version = "2.8.3" @@ -2626,7 +2465,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "metricsql_common" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ "chrono", ] @@ -2634,7 +2473,7 @@ dependencies = [ [[package]] name = "metricsql_parser" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=05fb36e4e9ff0b8e2ff6d43685785117ec0b9225#05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=1760a69f5f4431151317018a977b577e4e08c3d1#1760a69f5f4431151317018a977b577e4e08c3d1" dependencies = [ "ahash", "chrono", @@ -2661,14 +2500,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] -name = "miniz_oxide" -version = "0.9.1" +name = "minimal-lexical" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" @@ -2681,6 +2516,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2766,7 +2611,7 @@ dependencies = [ "futures-util", "http", "humantime", - "itertools", + "itertools 0.14.0", "parking_lot", "percent-encoding", "thiserror 2.0.20", @@ -2790,23 +2635,14 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "packedvec" version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" dependencies = [ + "bincode", "num-traits", - "serde", ] [[package]] @@ -2832,48 +2668,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "parquet" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", - "base64", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown 0.17.1", - "lz4_flex", - "num-bigint", - "num-integer", - "num-traits", - "object_store", - "paste", - "seq-macro", - "simdutf8", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -3008,6 +2802,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -3019,8 +2823,8 @@ dependencies = [ [[package]] name = "promql-parser" -version = "0.10.0" -source = "git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb#9fede7eecca923c9882fe256484d00d37f8706cb" +version = "0.11.1" +source = "git+https://github.com/ProjectASAP/promql-parser?rev=1d1562e91bbfd59eaceade219a9b98ebb07abdfe#1d1562e91bbfd59eaceade219a9b98ebb07abdfe" dependencies = [ "cfgrammar", "chrono", @@ -3047,7 +2851,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -3116,7 +2920,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3435,12 +3239,6 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - [[package]] name = "serde" version = "1.0.229" @@ -3517,17 +3315,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.11.3", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -3537,6 +3324,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -3553,12 +3346,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - [[package]] name = "simdutf8" version = "0.1.5" @@ -3605,10 +3392,10 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d" dependencies = [ + "bincode", "num-traits", "packedvec", - "serde", - "vob 3.0.6", + "vob", ] [[package]] @@ -3652,12 +3439,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" version = "0.11.1" @@ -3741,7 +3522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.52.0", @@ -3796,17 +3577,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "time" version = "0.3.55" @@ -4059,12 +3829,6 @@ dependencies = [ "rand 0.10.2", ] -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - [[package]] name = "unicode-ident" version = "1.0.24" @@ -4077,6 +3841,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -4089,6 +3859,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -4148,23 +3924,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "vob" -version = "3.0.6" +name = "virtue" +version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" -dependencies = [ - "num-traits", - "serde", -] +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" [[package]] name = "vob" -version = "4.0.1" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60681a44bd4be4b4da58435befb00f60f63372fb43a86cd977183114ad851198" +checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" dependencies = [ + "bincode", "num-traits", - "serde", ] [[package]] @@ -4556,12 +4328,6 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "zlib-rs" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" - [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 01679f19..3c7afdfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,8 @@ version = "0.1.0" # The backend depends on two Planner crates, at one immutable revision: # `asap-planner` to plan and `asap-executor` to execute. Planner IR types # are reached through `asap_executor::planner`. -asap-planner = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" } -asap-executor = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "05fb36e4e9ff0b8e2ff6d43685785117ec0b9225" } +asap-planner = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "1760a69f5f4431151317018a977b577e4e08c3d1" } +asap-executor = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "1760a69f5f4431151317018a977b577e4e08c3d1" } serde = { version = "1.0", features = ["derive"] } # `float_roundtrip`: a plan file reloads to the plan it was written from diff --git a/README.md b/README.md index 8a36243f..dafca1a7 100644 --- a/README.md +++ b/README.md @@ -10,126 +10,121 @@ Following the Planner's [planning stages](https://github.com/ProjectASAP/ASAPPlanner/blob/main/docs/design_docs/proposals/planner-layering.md), the Planner decides everything about a plan: which summaries to use, how to share them, what to materialize and which plan is cheapest. The backend -supplies its deployment inputs (`DeploymentCapabilities`, and Stage 3's -`PlanSelectionSettings`), then executes the one selected `PhysicalASAPDAG` as -given. +supplies its deployment inputs (`DeploymentCapabilities` and Stage 3's +`CostInputs`), then executes the one selected `PhysicalASAPDAG` as given, +with the Planner's executor (`asap-executor`). ```text -PlanningWorkload (JSON) DeploymentCapabilities + PlanSelectionSettings +PlanningWorkload (JSON) DeploymentCapabilities + CostInputs \ / control_plane: asap_planner::plan_installable | - plan.json: InstallablePlan (selected PhysicalASAPDAG, query per - root, stored outputs with retention, raw windows) + plan.json: InstallablePlan (selected PhysicalASAPDAG with each + view's and query-time Scan's retention_ms, query texts) + report.json: SelectedPlanReport (cost, predicted usage) | data_plane: Remote Write in, Prometheus query API out ``` -## Status +## How the data plane executes a plan + +The data plane keeps one time-indexed store, which the executor reads +through its `DataStore` interface: + +- **Raw data:** every Remote Write sample of a metric some `Scan` of the + plan reads, as a time series table `(ts, value, $promql_series_identity, + labels...)`. The DAG's own `TimeRange`s keep each window, an instant + selector's latest sample per series, and drop stale markers. +- **Materialized views:** each view's rows, appended by the ingestion run + that refreshed them with the `(start, end]` they cover, and kept for the + view's `retention_ms`. + +**Ingestion.** The plan lists its views refreshed at ingestion by refresh +group (`InstallablePlan::ingestion_refresh_groups`): views with one window +width `W` and phase `p`. A group's window `(e - W, e]`, `e = p (mod W)`, +closes when the watermark reaches `e`: the newest sample of a scanned metric, +capped at the wall clock, minus `--allowed-lateness-ms` (default 15 s). The +data plane then runs the group's views in one ingestion run over that window +and appends each view's rows. A window without samples is stored empty, not +missing. The first window of each group may have started before the data +plane did, so it is skipped, unless `--stream-start-ms` says where the +producers' stream starts. Windows that would already be dropped when the +watermark jumps are skipped. Raw samples are kept while a window still to be +refreshed or a query-time `Scan` (back to its `retention_ms`) reads them. +The plan's retention counts back from the newest evaluation, but ingestion +refreshes windows ahead of the queries it answers, so views and query-time +raw samples are kept `--view-retention-margin-ms` (default 60 s) longer: a +query up to that far behind the newest refreshed window is still answered. -The backend is being rebuilt on the Planner's physical DAG: - -| Part | State | -|---|---| -| Control plane: plan a PromQL workload, write `plan.json` | done | -| Data plane: load and validate `plan.json`, accept Remote Write | done | -| Stored summaries (SDS): one definition per computation, panes by end time | done | -| Precompute engine: build ingestion-time outputs once per tumbling pane | done | -| Query engine: answer planned instant queries from stored panes and raw samples | done | - -The data plane stores the plan's `stored_outputs`. Outputs that differ only -in which pane they read share one stored summary, kept for the largest -`retention_ms` the plan states among them (which covers range-evaluation -spans), plus one pane width so an evaluation one pane behind the newest -closed pane is still served; older panes are evicted. - -Panes close `--allowed-lateness-ms` (default 15 s) after the newest sample -of a stored summary's metric, capped at the wall clock, passes their end; later samples for them are -dropped. The first pane after startup is skipped because it may be partial. Samples more than `--max-future-ms` (default 5 min) ahead of the wall clock are dropped. Remote Write gets 503 when admitting it would leave more than -`--max-queued-samples` (default 2,000,000) samples waiting for the engine -(a request into an empty queue is always admitted), and 500 if the engine has +`--max-queued-samples` (default 2,000,000) samples waiting for the engine (a +request into an empty queue is always admitted), and 500 if the engine has stopped, which also makes `/-/ready` fail. `/api/v1/status/asap` reports the -engine's sample and pane counters as JSON. +engine's `ingestion` counters (samples accepted, late, future, unrouted; +windows refreshed, failed, skipped) as JSON. -A producer can close panes without waiting for later samples: +A producer can close windows without waiting for later samples: `POST /api/v1/precompute/watermark?time_ms=T` states that every sample at or before `T` has been written. After the writes acknowledged before it have -been ingested, every pane ending at or before `T` (capped at the wall clock) -is closed, so the last pane of a series that stopped sending is built and -served. A repeated or lower `T` does nothing and input stays open. A sample -that arrives later for a pane the barrier closed (one ending at or before -`T`) counts as late; a later sample in a pane still open is accepted, even if -it is at or before `T`. The response is `{"barrier_ms": B}`, the barrier in -force: `T` capped at the wall clock, or an earlier higher barrier. With several producers, -send the barrier only after all of them have written through `T`. - -An instant query whose text is one of the plan's queries is answered by the -plan (`x-asap-execution: plan`) at its evaluation time `T`. Stored outputs -built in an `IngestionRun` are read from the SDS as the panes ending at -`T - offset`, so `T` must be on the pane grid and those panes stored. The -plan's query-time `RawSampleWindow`s are read from Prometheus. A stored output -built in a `QueryRun` (`MatAtQueryKeptAcrossEvaluations`) is built from its -pane's raw samples the first time an evaluation needs it. It is stored in the -SDS under the same key once its pane is closed, that is, once its end is at or -before the watermark at the time the evaluation chose its fetch. Later -evaluations reuse it. It is evicted like an `IngestionRun` output: the plan's -`retention_ms` plus one pane, measured from the newest closed pane at or -before the watermark. `/api/v1/status/asap` reports `query_run` counters, -including summary states stored while their own metric lagged behind the -global watermark. Anything else is forwarded -to Prometheus with its end-to-end headers, such as `Authorization` and -`X-Scope-OrgID` (`x-asap-execution: forwarded`); a planned query that could not -be answered also says why in `x-asap-forward-reason`. Range queries, SQL and -ClickHouse are not supported yet. +been ingested, every window ending at or before `T` (capped at the wall +clock) is refreshed. A repeated or lower `T` does nothing. The response is +`{"barrier_ms": B}`, the barrier in force. With several producers, send the +barrier only after all of them have written through `T`. + +**Queries.** An instant query whose text is one of the plan's queries is +answered by one query run of its root at its evaluation time `T` +(`x-asap-execution: plan`). The run reads the views the root reaches, +through the windows its `TimeRange`s select, and the raw samples its +query-time `Scan`s read; every view read must find rows tiling its window, +or the query is not answered. Each run is limited to +`--run-memory-limit-bytes` (default 4 GiB). Anything else is forwarded to +Prometheus with its end-to-end headers, such as `Authorization` and +`X-Scope-OrgID` (`x-asap-execution: forwarded`); a planned query that could +not be answered also says why in `x-asap-forward-reason`. Runs execute one +at a time on the engine thread, after every write acknowledged before. + +Not supported yet: views refreshed on first read (so the backend declares +`query_time_results_retained: false`), range queries, SQL. ## Running ```sh # Plan #509 Example 1 with this backend's capabilities. cargo run -p control_plane -- \ - --workload examples/planner-layering-example1.json --out plan.json + --workload examples/planner-layering-example1.json --out plan.json --report report.json # Serve Remote Write on /api/v1/write and the query API on /api/v1/query[_range]. cargo run -p data_plane -- --plan plan.json --prometheus-url http://localhost:9090 ``` `--deployment-capabilities caps.json` replaces the default -`DeploymentCapabilities`: every summary the Planner can build, raw data kept -in Prometheus, and stored outputs built in a `QueryRun` (`MatAtQueryKeptAcrossEvaluations`). For example -`{"raw_data_retained": false}` makes the Planner build every input at -ingestion time. - -`--plan-selection-settings settings.json` replaces the Planner's default -`PlanSelectionSettings` (Stage 3's cost model, resource speed, measured -operation costs and `ignore_accuracy_guarantees`); an omitted field keeps its -default, cost model 1 with illustrative unit costs. -`--measured-operation-costs costs.json` gives Stage 3 the deployment's +`DeploymentCapabilities`: every summary the Planner's executor builds, raw +data kept by the data plane, and no query-time result kept across +evaluations. For example `{"raw_data_retained": false}` makes the Planner +build every input at ingestion time. + +`--cost-inputs costs.json` replaces the Planner's default `CostInputs` +(Stage 3's cost model, resource speed and measured operation costs); an +omitted field keeps its default, cost model 1 with illustrative unit costs. +`--measured-operation-costs table.json` gives Stage 3 the deployment's measured operation costs (`MeasuredOperationCosts`). `--ignore-accuracy-guarantees` accepts every query output whatever its -accuracy guarantee, so candidates are compared on cost alone; the plan's -`selection.queries_accepted_without_guarantee` lists the queries accepted -only because of it. - -`--stream-start-ms T` declares that the producers write every raw sample at -or after `T`: the first pane of each stored output that starts at or after -`T` is complete and is built. Without it, the first pane each stored output -sees may have started before the data plane did, so it is skipped and a -query whose window needs it is not answered from the plan. +accuracy guarantee, so candidates are compared on cost alone. +`--report report.json` writes Stage 3's `SelectedPlanReport`: the plan's +cost and predicted usage per node and per query. The data facts the plan +assumed are printed. The Planner returns a plan only when its executor +compiles it; otherwise the control plane fails. `--forward-timeout-seconds` bounds each forwarded query (default 120, matching Prometheus's default query timeout); a query that runs longer gets 502. `--no-forward` makes every answer ASAP's: the data plane never sends a query -to Prometheus for computation. It refuses to start with a plan that has a -query it cannot run, and answers any query the plan does not answer (not -planned, a pane not stored, an evaluation time off the pane grid, a range -query) with a Prometheus `execution` error (422, `x-asap-execution: refused`) -whose `error` is the reason a forwarded query's `x-asap-forward-reason` would -give. Raw samples for query-time windows are still read from Prometheus. The -control plane writes a plan only when every workload query has one the data -plane can execute; otherwise it fails. +to Prometheus. It refuses to start with a plan that has a query it cannot +run, and answers any query the plan does not answer (not planned, a window +not stored, a range query) with a Prometheus `execution` error (422, +`x-asap-execution: refused`) whose `error` is the reason a forwarded query's +`x-asap-forward-reason` would give. ## Development @@ -139,5 +134,5 @@ cargo clippy --workspace --all-targets --locked -- -D warnings cargo test --workspace --locked ``` -The Planner revision is pinned in `Cargo.toml`; see +The Planner revision is pinned in `Cargo.toml` (ASAPPlanner `1760a69f`); see [scripts/ci/README.md](scripts/ci/README.md). diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 5e575d53..74c46a88 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -3,27 +3,30 @@ //! //! The Planner runs every planning stage, including plan selection, and //! returns the selected DAG ready to install ([`InstallablePlan`]): the DAG, -//! the query each root answers, the outputs to store with how long each -//! must stay available, and the raw-sample windows read at query time. This -//! crate only supplies the deployment inputs: what the data plane can -//! execute and keep, and how Stage 3 prices candidates. +//! with each materialized view's and query-time `Scan`'s `retention_ms`, +//! and the query each root answers. This crate only supplies the +//! deployment inputs: what the data plane can execute and keep, and how +//! Stage 3 costs candidates. use std::path::Path; use asap_planner::{ - plan_installable, DeploymentCapabilities, FrontendInput, InstallablePlan, InstallablePlanError, - PlanSelectionSettings, PlanningModels, PlanningWorkload, UserInput, + plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, + InstallablePlanError, PlanningModels, PlanningWorkload, SelectedPlanReport, UserInput, }; /// What the backend gives the Planner about its deployment. The Planner /// selects; these only say what can run and what it costs. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Default)] pub struct DeploymentInputs { /// What the data plane can execute and keep (Stages 1–3). pub deployment_capabilities: DeploymentCapabilities, - /// How Stage 3 prices candidates: cost model, resource speed, and any + /// How Stage 3 costs candidates: cost model, resource speed, and any /// measured operation costs. - pub plan_selection: PlanSelectionSettings, + pub cost_inputs: CostInputs, + /// Accept every query output regardless of its accuracy guarantee: + /// candidates are compared on cost alone. + pub ignore_accuracy_guarantees: bool, } #[derive(Debug, thiserror::Error)] @@ -32,75 +35,72 @@ pub enum PlanError { Planning(#[from] asap_planner::PlanError), #[error("invalid plan: {0}")] InvalidPlan(#[from] InstallablePlanError), + #[error("plan the executor cannot run: {0}")] + NotExecutable(String), #[error("{path}: {message}")] File { path: String, message: String }, } /// Plans `workload` with the Planner (frontend and Stages 1–3) and returns -/// the plan Stage 3 selected, ready to install. Only PromQL workloads are -/// planned: the Planner refuses another language for the PromQL frontend. -/// It rejects every candidate that needs more than -/// `inputs.deployment_capabilities`, and refuses a plan a runtime could not -/// run through the executor. +/// the plan Stage 3 selected, ready to install, with Stage 3's report of +/// it. Only PromQL workloads are planned. The Planner rejects every +/// candidate that needs more than `inputs.deployment_capabilities` and +/// refuses a plan its executor cannot compile. pub fn plan_workload( workload: &PlanningWorkload, inputs: &DeploymentInputs, -) -> Result { +) -> Result<(InstallablePlan, SelectedPlanReport), PlanError> { let models = PlanningModels::builtin() .with_deployment_capabilities(&inputs.deployment_capabilities) - .with_plan_selection(&inputs.plan_selection); - let input = UserInput::new(workload, FrontendInput::Promql { histograms: None }, models); + .with_cost_inputs(&inputs.cost_inputs) + .with_ignore_accuracy_guarantees(inputs.ignore_accuracy_guarantees); + let input = UserInput::new(workload, FrontendInput::Promql, models); // Only the SQL frontend awaits anything; PromQL planning completes on // its first poll. - let plan = futures::executor::block_on(plan_installable(input))?; - if let Some(selection) = &plan.selection { - tracing::info!(candidate = %selection.selected_candidate_id, "selected physical plan"); - } - Ok(plan) + Ok(futures::executor::block_on(plan_installable(input))?) } /// Writes `plan` as the plan file the data plane installs. pub fn save_plan(plan: &InstallablePlan, path: &Path) -> Result<(), PlanError> { - let text = serde_json::to_string_pretty(plan).expect("plans serialize"); + write_json(plan, path) +} + +/// Writes Stage 3's report of the selected plan (cost and predicted usage). +pub fn save_report(report: &SelectedPlanReport, path: &Path) -> Result<(), PlanError> { + write_json(report, path) +} + +fn write_json(value: &impl serde::Serialize, path: &Path) -> Result<(), PlanError> { + let text = serde_json::to_string_pretty(value).expect("plans serialize"); std::fs::write(path, text).map_err(|e| PlanError::File { path: path.display().to_string(), message: e.to_string(), }) } -/// Reads and validates a plan written by [`save_plan`]. +/// Reads a plan written by [`save_plan`] and checks that it is valid and +/// that the Planner's executor compiles it. pub fn load_plan(path: &Path) -> Result { let file_error = |message: String| PlanError::File { path: path.display().to_string(), message, }; let text = std::fs::read_to_string(path).map_err(|e| file_error(e.to_string()))?; - let plan: InstallablePlan = - serde_json::from_str(&text).map_err(|e| file_error(e.to_string()))?; + let plan = InstallablePlan::from_json(&text)?; plan.validate()?; + plan.check_compiles().map_err(PlanError::NotExecutable)?; Ok(plan) } /// What this backend's data plane can execute and keep: every summary the -/// Planner executor builds, raw samples kept in Prometheus, and stored -/// outputs built in a `QueryRun` (`MatAtQueryKeptAcrossEvaluations`): the -/// query engine builds their summary states on first use and stores them in -/// the SDS once their coverage is closed. +/// Planner's executor builds (the Planner's default when left `None`) and +/// raw samples, kept in its store for query-time `Scan`s. It refreshes no +/// view on first read yet, so no query-time result is kept across +/// evaluations. pub fn backend_capabilities() -> DeploymentCapabilities { DeploymentCapabilities { supported_summaries: None, raw_data_retained: true, - query_time_results_retained: true, - } -} - -impl Default for DeploymentInputs { - /// [`backend_capabilities`] with the Planner's default plan selection - /// settings (cost model 1 with illustrative unit costs). - fn default() -> Self { - Self { - deployment_capabilities: backend_capabilities(), - plan_selection: PlanSelectionSettings::default(), - } + query_time_results_retained: false, } } diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 76b77a91..6b33cde3 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use asap_planner::PlanningWorkload; use clap::Parser; -use control_plane::{plan_workload, save_plan, DeploymentInputs}; +use control_plane::{plan_workload, save_plan, save_report, DeploymentInputs}; /// Plan a workload with ASAPPlanner and write the selected plan. #[derive(Parser)] @@ -15,26 +15,28 @@ struct Args { /// backend's data plane executes and keeps. #[arg(long)] deployment_capabilities: Option, - /// Planner `PlanSelectionSettings` (JSON): Stage 3's cost model, - /// resource speed, measured operation costs and whether accuracy - /// guarantees are ignored; an omitted field takes the Planner's + /// Planner `CostInputs` (JSON): Stage 3's cost model, resource speed + /// and measured operation costs; an omitted field takes the Planner's /// default. #[arg(long)] - plan_selection_settings: Option, - /// Planner `MeasuredOperationCosts` (JSON, or the earlier list of - /// `MeasuredOperationCost`) that Stage 3 prices operators from, in place - /// of the settings' table; by default every node is priced from the - /// Planner's formulas. + cost_inputs: Option, + /// Planner `MeasuredOperationCosts` (JSON) that Stage 3 costs operators + /// from, in place of the cost inputs' table; by default every node is + /// costed from the Planner's formulas. #[arg(long)] measured_operation_costs: Option, - /// Accept every query output regardless of its accuracy guarantee - /// (`PlanSelectionSettings::ignore_accuracy_guarantees`): candidates are - /// compared on cost alone; executability checks still apply. + /// Accept every query output regardless of its accuracy guarantee: + /// candidates are compared on cost alone; executability checks still + /// apply. #[arg(long)] ignore_accuracy_guarantees: bool, /// Where to write the plan the data plane installs. #[arg(long)] out: PathBuf, + /// Where to write Stage 3's report of the selected plan: its cost and + /// predicted resource usage, per node and per query. + #[arg(long)] + report: Option, } fn read_json(path: &Path) -> anyhow::Result { @@ -53,28 +55,26 @@ fn main() -> anyhow::Result<()> { if let Some(path) = &args.deployment_capabilities { inputs.deployment_capabilities = read_json(path)?; } - if let Some(path) = &args.plan_selection_settings { - inputs.plan_selection = read_json(path)?; + if let Some(path) = &args.cost_inputs { + inputs.cost_inputs = read_json(path)?; } if let Some(path) = &args.measured_operation_costs { - inputs.plan_selection.measured_operation_costs = read_json(path)?; + inputs.cost_inputs.measured_operation_costs = read_json(path)?; } - inputs.plan_selection.ignore_accuracy_guarantees |= args.ignore_accuracy_guarantees; - let plan = plan_workload(&workload, &inputs)?; + inputs.ignore_accuracy_guarantees = args.ignore_accuracy_guarantees; + let (plan, report) = plan_workload(&workload, &inputs)?; save_plan(&plan, &args.out)?; - if let Some(selection) = &plan.selection { - if !selection.queries_accepted_without_guarantee.is_empty() { - println!( - "queries accepted without an accuracy guarantee (workload indices): {:?}", - selection.queries_accepted_without_guarantee - ); - } + if let Some(path) = &args.report { + save_report(&report, path)?; + } + for fact in &report.assumed_data_facts { + println!("assumed data fact: {fact:?}"); } println!( "wrote {} ({} queries, {} nodes)", args.out.display(), - plan.queries.len(), - plan.dag.dag.nodes.len() + plan.query_texts.len(), + plan.dag.nodes.len() ); Ok(()) } diff --git a/control_plane/tests/plan_workload.rs b/control_plane/tests/plan_workload.rs index a9bbbd2c..fe47cf0a 100644 --- a/control_plane/tests/plan_workload.rs +++ b/control_plane/tests/plan_workload.rs @@ -1,5 +1,6 @@ use std::path::{Path, PathBuf}; +use asap_planner::{InstallablePlan, MaterializedViewRefresh}; use asap_planner::{InstallablePlanError, PlanningWorkload}; use control_plane::{load_plan, plan_workload, save_plan, DeploymentInputs, PlanError}; @@ -9,6 +10,10 @@ fn example(name: &str) -> PlanningWorkload { serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() } +fn plan(workload: &PlanningWorkload, inputs: &DeploymentInputs) -> InstallablePlan { + plan_workload(workload, inputs).unwrap().0 +} + fn temp_path(name: &str) -> PathBuf { std::env::temp_dir().join(format!("control-plane-{}-{name}", std::process::id())) } @@ -18,49 +23,45 @@ fn temp_path(name: &str) -> PathBuf { #[test] fn plans_example1_and_round_trips_the_plan() { let workload = example("example1"); - let plan = plan_workload(&workload, &DeploymentInputs::default()).unwrap(); + let plan = plan(&workload, &DeploymentInputs::default()); assert_eq!( - plan.queries - .iter() - .map(|q| q.query.clone()) - .collect::>(), + plan.query_texts, workload .query_workload .entries() .map(|entry| entry.query.0) .collect::>() ); - assert_eq!(plan.dag.dag.roots.len(), 2); + assert_eq!(plan.dag.roots.len(), 2); let path = temp_path("example1.json"); save_plan(&plan, &path).unwrap(); - assert_eq!(load_plan(&path).unwrap(), plan); + // The plan file carries what a runtime installs; in-memory provenance + // (`selected_physical_candidate`) is not part of it. + assert_eq!( + serde_json::to_value(load_plan(&path).unwrap()).unwrap(), + serde_json::to_value(&plan).unwrap() + ); std::fs::remove_file(path).unwrap(); } -/// Example 4b without raw data stores its six 10-minute panes, built at -/// ingestion, each kept as long as the Planner states: 60 minutes back for -/// the oldest pane the 60-minute range reads. +/// Example 4b without raw data keeps its 60-minute range as a chain of +/// tumbling windows refreshed at ingestion, retained for the 60 minutes the +/// range reads back, and scans no raw data at query time. #[test] -fn example4b_stores_panes_with_the_planners_retention() { +fn example4b_keeps_tumbling_windows_with_the_planners_retention() { let mut inputs = DeploymentInputs::default(); inputs.deployment_capabilities.raw_data_retained = false; - let plan = plan_workload(&example("example4b"), &inputs).unwrap(); - assert_eq!(plan.stored_outputs.len(), 6); - let mut offsets: Vec = plan - .stored_outputs + let plan = plan(&example("example4b"), &inputs); + let views = plan.materialized_views().unwrap(); + assert!(!views.is_empty()); + assert!(views .iter() - .map(|output| { - let [window] = output.inputs.raw_sample_windows.as_slice() else { - panic!("one raw-sample window: {output:?}"); - }; - assert_eq!(window.width_ms, 600_000); - window.end_offset_ms - }) - .collect(); - offsets.sort(); - assert_eq!(offsets, (0..6).map(|i| i * 600_000).collect::>()); - let retention = plan.stored_outputs.iter().map(|o| o.retention_ms).max(); - assert_eq!(retention, Some(3_600_000)); + .all(|v| v.refresh() == MaterializedViewRefresh::AtIngestion)); + assert_eq!( + views.iter().map(|v| v.retention_ms()).max(), + Some(3_600_000) + ); + assert!(plan.raw_data_retention().unwrap().is_empty()); } /// Only PromQL workloads are planned; the Planner's PromQL frontend refuses @@ -79,8 +80,8 @@ fn rejects_sql_workloads() { /// A plan whose queries do not match the DAG's roots is not installed. #[test] fn load_rejects_a_plan_without_a_query_per_root() { - let mut plan = plan_workload(&example("example1"), &DeploymentInputs::default()).unwrap(); - plan.queries.pop(); + let mut plan = plan(&example("example1"), &DeploymentInputs::default()); + plan.query_texts.pop(); let path = temp_path("mismatched.json"); save_plan(&plan, &path).unwrap(); assert!(matches!( @@ -92,26 +93,19 @@ fn load_rejects_a_plan_without_a_query_per_root() { std::fs::remove_file(path).unwrap(); } -/// The backend retains query-time results (`query_time_results_retained`), -/// so for a few series sampled every millisecond the Planner selects #509 -/// Example 4's B3: five 1-minute KLL panes, stored outputs built in a -/// `QueryRun` (`MatAtQueryKeptAcrossEvaluations`). +/// The data plane refreshes no view on first read, so the backend keeps +/// no query-time result across evaluations: #509 Example 4's B3 workload +/// gets no view refreshed on first read. #[test] -fn selects_stored_outputs_built_in_a_query_run() { +fn plans_no_view_refreshed_on_first_read() { let inputs = DeploymentInputs::default(); - assert!(inputs.deployment_capabilities.query_time_results_retained); - let plan = plan_workload(&example("example4-b3"), &inputs).unwrap(); - let query_run: Vec<_> = plan - .stored_outputs - .iter() - .filter(|o| o.built_at == asap_planner::StoredOutputBuild::QueryRun) - .collect(); - assert_eq!(query_run.len(), 5, "five one-minute summaries"); - assert!(query_run.iter().all(|o| o - .inputs - .raw_sample_windows + assert!(!inputs.deployment_capabilities.query_time_results_retained); + let plan = plan(&example("example4-b3"), &inputs); + assert!(plan + .materialized_views() + .unwrap() .iter() - .all(|w| w.width_ms == 60_000))); + .all(|v| v.refresh() != MaterializedViewRefresh::OnFirstRead)); } /// A workload is planned whole or refused: when one query has no plan the @@ -134,16 +128,14 @@ fn refuses_a_workload_with_a_query_it_cannot_plan() { /// The MVP benchmark's p99 query over a gauge not declared non-negative: /// without raw data, every candidate fails a check (the exact ones scan raw /// samples at query time, the merged DDSketch panes carry no bound through -/// `avg`), so planning fails. With `ignore_accuracy_guarantees`, loaded from -/// the Planner's settings JSON as `--plan-selection-settings` reads it, the -/// cheapest sketch plan is selected and the report marks the query as -/// accepted without a guarantee. +/// `avg`), so planning fails. With `ignore_accuracy_guarantees` the +/// cheapest sketch plan is selected: tumbling windows kept at ingestion. #[test] fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { let workload: PlanningWorkload = serde_json::from_str( r#"{"query_workload": {"language": "promql", "repeating_queries": [{ "query": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", - "demand": {"fixed_interval": 1000}, + "demand": {"fixed_interval_at": {"interval": 1000, "evaluation_phase": 0}}, "requirements": {"accuracy": {"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}, "predictability": {"predictable": {"known_at": null}}}]}, @@ -157,10 +149,7 @@ fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { let error = plan_workload(&workload, &inputs).unwrap_err(); assert!(matches!(error, PlanError::Planning(_)), "{error}"); - inputs.plan_selection = - serde_json::from_str(r#"{"ignore_accuracy_guarantees": true}"#).unwrap(); - let plan = plan_workload(&workload, &inputs).unwrap(); - assert!(!plan.stored_outputs.is_empty(), "panes built at ingestion"); - let selection = plan.selection.as_ref().unwrap(); - assert_eq!(selection.queries_accepted_without_guarantee, [0]); + inputs.ignore_accuracy_guarantees = true; + let plan = plan(&workload, &inputs); + assert!(!plan.materialized_views().unwrap().is_empty()); } diff --git a/data_plane/src/engine.rs b/data_plane/src/engine.rs index 51978bf6..e84fe740 100644 --- a/data_plane/src/engine.rs +++ b/data_plane/src/engine.rs @@ -1,28 +1,26 @@ //! The engine thread: owns the installed plan, compiled once when it is -//! installed, and everything that executes it; handlers send the engine -//! commands. The compiled plan is `Send`, so it is compiled on the -//! installing thread and moved to the engine thread. +//! installed, the store, and everything that executes the plan; handlers +//! send the engine commands. Runs execute one at a time, so the store +//! changes only between runs. -use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::{Arc, Mutex}; +use asap_executor::runtime::Limits; use asap_planner::InstallablePlan; use tokio::sync::oneshot; -use crate::precompute::{Precompute, PrecomputeOptions, PrecomputeStats, RawSample}; -use crate::query::{ - IngestionProgress, Prepared, QueryEngine, QueryRunStats, ResultSample, Unanswered, -}; -use crate::sds::Sds; +use crate::ingestion::{Ingestion, IngestionOptions, IngestionStats}; +use crate::query::{QueryEngine, ResultSample, Unanswered}; +use crate::store::{scanned_metrics, RawSample, TimeIndexedStore}; -/// How the engine thread ingests samples. +/// How the engine thread ingests samples and runs queries. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct EngineOptions { - /// Lateness, future bound and pane checks of the precompute engine. - pub precompute: PrecomputeOptions, + /// Lateness, future bound and first-window checks of ingestion. + pub ingestion: IngestionOptions, /// Samples queued for the engine, not yet ingested, beyond which Remote /// Write is refused with 503 so senders retry. pub max_queued_samples: u64, @@ -30,15 +28,19 @@ pub struct EngineOptions { /// query engine cannot run is refused at install, and the data plane /// never forwards a query to Prometheus (`--no-forward`). pub no_forward: bool, + /// The most bytes one ingestion or query run may hold + /// (`asap_executor::runtime::Limits::max_bytes`). + pub run_memory_limit_bytes: usize, } impl Default for EngineOptions { fn default() -> Self { Self { - precompute: PrecomputeOptions::default(), + ingestion: IngestionOptions::default(), // Two Remote Write requests of the default sample limit. max_queued_samples: 2_000_000, no_forward: false, + run_memory_limit_bytes: 4 << 30, } } } @@ -47,16 +49,9 @@ type Reply = oneshot::Sender>; enum Command { Ingest(Vec), - BoundRawSampleWindows { - query: String, - time_ms: i64, - reply: Reply, - }, Evaluate { - root: usize, + query: String, time_ms: i64, - progress: IngestionProgress, - raw: BTreeMap>, reply: Reply>, }, AdvanceWatermark { @@ -86,10 +81,8 @@ pub struct EngineHandle { max_queued_samples: u64, /// False once the engine thread has exited, for any reason. running: Arc, - /// The counters as of the last command the engine ran. - stats: Arc>, - /// The stored output built in a `QueryRun` counters as of the last evaluation. - query_run_stats: Arc>, + /// The ingestion counters as of the last command the engine ran. + stats: Arc>, } /// Marks the engine stopped when its thread exits, panics included. @@ -102,40 +95,48 @@ impl Drop for RunningGuard { } impl EngineHandle { - /// Starts the engine for the plan serialized in `plan_json`, parsed and - /// compiled once, and returns the plan's queries. Fails if the - /// plan is invalid or the data plane cannot execute it. + /// Starts the engine for the plan file `plan_json`, read, checked and + /// compiled once on the engine thread, and returns the plan's queries. + /// Fails if the plan is invalid or the data plane cannot execute it. pub fn spawn(plan_json: String, options: EngineOptions) -> anyhow::Result<(Self, Vec)> { let (commands, receiver) = channel(); - let (engine, queries) = Engine::new(&plan_json, options)?; + let (installed, installation) = channel(); let running = Arc::new(AtomicBool::new(true)); let queued_samples = Arc::new(AtomicU64::new(0)); - let stats = Arc::new(Mutex::new(PrecomputeStats::default())); - let query_run_stats = Arc::new(Mutex::new(QueryRunStats::default())); + let stats = Arc::new(Mutex::new(IngestionStats::default())); let shared = Shared { queued_samples: queued_samples.clone(), stats: stats.clone(), - query_run_stats: query_run_stats.clone(), }; let guard = RunningGuard(running.clone()); std::thread::Builder::new() .name("engine".into()) .spawn(move || { let _guard = guard; - engine.serve(receiver, shared); + match Engine::new(&plan_json, options) { + Ok((engine, queries)) => { + let _ = installed.send(Ok(queries)); + engine.serve(receiver, shared); + } + Err(error) => { + let _ = installed.send(Err(error)); + } + } })?; + let queries = installation + .recv() + .map_err(|_| anyhow::anyhow!("the engine thread exited while installing"))??; let handle = Self { commands, queued_samples, max_queued_samples: options.max_queued_samples, running, stats, - query_run_stats, }; Ok((handle, queries)) } - /// Queues `samples` for precompute. `Busy` when the queue would hold + /// Queues `samples` for ingestion. `Busy` when the queue would hold /// more than `max_queued_samples`; `Stopped` when the engine has exited. pub fn try_ingest(&self, samples: Vec) -> Result<(), IngestError> { if !self.is_running() { @@ -161,51 +162,22 @@ impl EngineHandle { }) } - /// The plan root answering `query` at `time_ms` and the raw samples it - /// needs; see [`QueryEngine::bound_raw_sample_windows`]. - pub async fn bound_raw_sample_windows( - &self, - query: String, - time_ms: i64, - ) -> Result { - self.ask(|reply| Command::BoundRawSampleWindows { - query, - time_ms, - reply, - }) - .await - } - - /// Runs a prepared root over stored summaries and `raw`; see - /// [`QueryEngine::evaluate`]. + /// Answers the planned `query` at `time_ms`, after every write + /// acknowledged before; see [`QueryEngine::evaluate`]. pub async fn evaluate( &self, - prepared: Prepared, + query: String, time_ms: i64, - raw: BTreeMap>, ) -> Result, Unanswered> { - self.ask(|reply| Command::Evaluate { - root: prepared.root, - time_ms, - progress: prepared.progress, - raw, - reply, - }) - .await - } - - /// The stored output built in a `QueryRun` counters as of the last evaluation. - pub fn query_run_stats(&self) -> QueryRunStats { - *self - .query_run_stats - .lock() - .unwrap_or_else(|e| e.into_inner()) - } - - async fn ask(&self, command: impl FnOnce(Reply) -> Command) -> Result { let stopped = || Unanswered::Execution("the engine has stopped".into()); let (reply, answer) = oneshot::channel(); - self.commands.send(command(reply)).map_err(|_| stopped())?; + self.commands + .send(Command::Evaluate { + query, + time_ms, + reply, + }) + .map_err(|_| stopped())?; answer.await.map_err(|_| stopped())? } @@ -219,15 +191,15 @@ impl EngineHandle { self.queued_samples.load(Ordering::SeqCst) } - /// The precompute counters as of the last batch the engine ingested. - pub fn stats(&self) -> PrecomputeStats { + /// The ingestion counters as of the last command the engine ran. + pub fn stats(&self) -> IngestionStats { *self.stats.lock().unwrap_or_else(|e| e.into_inner()) } /// Queues an event-time barrier behind every write already acknowledged - /// and waits until it has run: every pane ending at or before - /// `barrier_ms` (capped at the wall clock) is then closed. Returns the - /// barrier in force; see [`Precompute::advance_watermark_at`]. + /// and waits until it has run: every window ending at or before + /// `barrier_ms` (capped at the wall clock) is then refreshed. Returns + /// the barrier in force; see [`Ingestion::advance_watermark_at`]. pub async fn advance_watermark(&self, barrier_ms: i64) -> Result { if !self.is_running() { return Err(IngestError::Stopped); @@ -265,27 +237,30 @@ fn admit(queued: u64, count: u64, max_queued: u64) -> Result { /// What the engine thread publishes to the HTTP side. struct Shared { queued_samples: Arc, - stats: Arc>, - query_run_stats: Arc>, + stats: Arc>, } struct Engine { - precompute: Precompute, + store: TimeIndexedStore, + ingestion: Ingestion, query: QueryEngine, } impl Engine { fn new(plan_json: &str, options: EngineOptions) -> anyhow::Result<(Self, Vec)> { - let plan: InstallablePlan = serde_json::from_str(plan_json)?; + let plan = InstallablePlan::from_json(plan_json)?; plan.validate()?; - let sds = Sds::from_plan(&plan)?; + let views = plan.materialized_views()?; + let groups = plan.ingestion_refresh_groups()?; + let limits = Limits { + max_bytes: options.run_memory_limit_bytes, + }; tracing::info!( - definitions = sds.definitions.len(), - outputs = sds.outputs.len(), - "stored summaries" + views = views.len(), + refresh_groups = groups.len(), + "materialized views" ); - let queries = plan.queries.iter().map(|q| q.query.clone()).collect(); - let query = QueryEngine::new(&plan, &sds); + let query = QueryEngine::new(&plan, &views, limits.clone()); let unexecutable = query.unexecutable(); if options.no_forward && !unexecutable.is_empty() { let reasons: Vec = unexecutable @@ -297,60 +272,40 @@ impl Engine { reasons.join("; ") ); } - let engine = Self { - precompute: Precompute::new(plan.dag.dag, &sds, options.precompute)?, - query, - }; - Ok((engine, queries)) + let ingestion = Ingestion::new(&plan.dag, &views, groups, options.ingestion, limits)?; + let store = TimeIndexedStore::for_metrics(&scanned_metrics(&plan.dag.nodes)); + let queries = plan.query_texts.clone(); + Ok(( + Self { + store, + ingestion, + query, + }, + queries, + )) } fn serve(mut self, commands: Receiver, shared: Shared) { for command in commands { match command { Command::Ingest(samples) => { - self.precompute.ingest(&samples); - *shared.stats.lock().unwrap_or_else(|e| e.into_inner()) = self.precompute.stats; - shared - .queued_samples - .fetch_sub(samples.len() as u64, Ordering::SeqCst); - } - Command::BoundRawSampleWindows { - query, - time_ms, - reply, - } => { - let _ = reply.send(self.query.bound_raw_sample_windows( - &query, - time_ms, - &self.precompute.store, - self.precompute.progress(), - )); + let count = samples.len() as u64; + self.ingestion.ingest(&mut self.store, samples); + *shared.stats.lock().unwrap_or_else(|e| e.into_inner()) = self.ingestion.stats; + shared.queued_samples.fetch_sub(count, Ordering::SeqCst); } Command::Evaluate { - root, + query, time_ms, - progress, - raw, reply, } => { - let answer = self.query.evaluate( - root, - time_ms, - &mut self.precompute.store, - &progress, - &raw, - ); - // Published before the reply, so a caller that awaited - // the evaluation reads its counters. - *shared - .query_run_stats - .lock() - .unwrap_or_else(|e| e.into_inner()) = self.query.query_run_stats; - let _ = reply.send(answer); + let _ = reply.send(self.query.evaluate(&query, time_ms, &self.store)); } Command::AdvanceWatermark { barrier_ms, reply } => { - let barrier_ms = self.precompute.advance_watermark(barrier_ms); - *shared.stats.lock().unwrap_or_else(|e| e.into_inner()) = self.precompute.stats; + let barrier_ms = self + .ingestion + .advance_watermark(&mut self.store, barrier_ms); + *shared.stats.lock().unwrap_or_else(|e| e.into_inner()) = self.ingestion.stats; let _ = reply.send(barrier_ms); } Command::Stop => break, diff --git a/data_plane/src/ingestion.rs b/data_plane/src/ingestion.rs new file mode 100644 index 00000000..622289aa --- /dev/null +++ b/data_plane/src/ingestion.rs @@ -0,0 +1,551 @@ +//! Ingestion: stores raw samples and refreshes the plan's materialized +//! views at ingestion time, by refresh group, one ingestion run per +//! refresh window. +//! +//! The plan lists its views refreshed at ingestion by refresh group +//! (`InstallablePlan::ingestion_refresh_groups`): the views whose windows +//! have one width `W` and end at one phase `p` modulo `W`. A group's window +//! `(e - W, e]` (`e ≡ p mod W`) closes when the watermark (the newest +//! sample time minus the allowed lateness, or a producer's barrier) reaches +//! `e`; the engine then runs the group's views in one ingestion run over +//! that window and appends each view's rows to the store. A window with no +//! sample is refreshed too, so its rows are stored as empty, not missing. +//! +//! The first window a group sees may have started before the data plane +//! did, so it is never refreshed and queries that read it are not +//! answered, unless the producer declares where its stream starts +//! (`stream_start_ms`): a window starting at or after it is complete. +//! When the watermark jumps past a view's retention, the windows that would +//! already be dropped are skipped. +//! +//! Raw samples are kept while a window still to be refreshed or a +//! query-time `Scan` (back to its `retention_ms`) reads them. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use asap_executor::physical_planner::{compile, CompiledPhysicalDAG}; +use asap_executor::planner::ir::operator::Source; +use asap_executor::planner::ir::{NonASAPOp, Operator}; +use asap_executor::planner::plan::materialized_view::{ + IngestionRefreshGroup, MaterializedView, MaterializedViewRefresh, +}; +use asap_executor::planner::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_executor::runtime::{Limits, RunContext, Scope}; +use futures::executor::block_on; + +use crate::store::{RawSample, TimeIndexedStore}; + +/// How ingestion treats sample times. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IngestionOptions { + /// A window closes when the newest routed sample is this far past its end. + pub allowed_lateness_ms: u64, + /// Samples further ahead of the wall clock than this are dropped. + pub max_future_ms: u64, + /// Where the producer's stream starts: every raw sample at or after it + /// reaches the data plane, so a group's first window that starts at or + /// after it is complete and is refreshed. `None`: the first window may + /// have started before the data plane did, so it is never refreshed. + pub stream_start_ms: Option, + /// How much longer than its `retention_ms` a view's rows, and a + /// query-time `Scan`'s raw samples, are kept. The plan's retention + /// counts back from the newest evaluation, but ingestion runs ahead of + /// the queries it answers: a query at `T` reads windows ending up to + /// `retention_ms` before `T` while newer windows have already been + /// refreshed. With this margin a query up to `view_retention_margin_ms` + /// behind the newest refreshed window is still answered. + pub view_retention_margin_ms: u64, +} + +impl Default for IngestionOptions { + fn default() -> Self { + Self { + allowed_lateness_ms: 15_000, + max_future_ms: 5 * 60_000, + stream_start_ms: None, + view_retention_margin_ms: 60_000, + } + } +} + +/// Counters of what happened to samples (each counted once) and to the +/// refresh windows of each group (each counted once). +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize)] +pub struct IngestionStats { + /// Samples stored as raw data. + pub samples_accepted: u64, + /// Samples older than every window still to be refreshed and every + /// query-time `Scan`'s retention. + pub samples_dropped_late: u64, + /// Samples more than `max_future_ms` ahead of the wall clock. + pub samples_dropped_future: u64, + /// Samples of a metric no `Scan` reads. + pub samples_unrouted: u64, + /// Ingestion runs that refreshed a group's window and stored its rows. + pub windows_refreshed: u64, + /// Ingestion runs that failed; their window is not stored. + pub windows_failed: u64, + /// First windows not refreshed because they may have started before the + /// data plane did. + pub windows_skipped_first: u64, + /// Windows not refreshed because they would already be dropped. + pub windows_skipped_expired: u64, +} + +/// Why a plan cannot be ingested. +#[derive(Debug, thiserror::Error)] +pub enum IngestionError { + #[error("{0}")] + Unsupported(String), + #[error("refresh group of views {views:?}: {error}")] + Compile { + views: Vec, + error: asap_executor::Error, + }, +} + +/// One refresh group, compiled once. +struct RefreshGroupRun { + width_ms: i64, + phase_ms: i64, + /// The group's views, with `retention_ms` extended by + /// `view_retention_margin_ms`: how long the store keeps their rows. + views: Vec, + compiled: CompiledPhysicalDAG, + /// The metrics its runs scan. + metrics: BTreeSet, + /// The longest retention of its views. + retention_ms: i64, + /// End of the newest window refreshed or skipped; `None` before the + /// first sample of a metric it scans. + refreshed_through: Option, +} + +impl RefreshGroupRun { + /// The end of the group's window holding `timestamp_ms`. + fn window_end(&self, timestamp_ms: i64) -> i64 { + window_end(timestamp_ms - self.phase_ms, self.width_ms) + self.phase_ms + } +} + +pub struct Ingestion { + groups: Vec, + /// For each metric a query-time `Scan` reads, how far back it reads + /// (`None`: every sample). + query_time_lookback_ms: HashMap>, + options: IngestionOptions, + limits: Limits, + newest_sample_ms: Option, + /// The newest wall-clock time an ingest ran at: the watermark never + /// passes it, so a series stamped ahead of the others cannot close + /// windows early. + newest_clock_ms: Option, + /// The newest event-time barrier a producer declared. + barrier_ms: Option, + pub stats: IngestionStats, +} + +impl Ingestion { + /// Compiles one ingestion run per refresh group of `dag`'s `views`. + /// Refuses views refreshed on first read, which the data plane does not + /// refresh yet, and groups without a bounded window. + pub fn new( + dag: &PhysicalASAPDAG, + views: &[MaterializedView], + groups: Vec, + options: IngestionOptions, + limits: Limits, + ) -> Result { + if let Some(view) = views + .iter() + .find(|v| v.refresh() == MaterializedViewRefresh::OnFirstRead) + { + return Err(IngestionError::Unsupported(format!( + "materialized view {} is refreshed on first read, which the data plane does not do yet", + view.node_id + ))); + } + let index = DagIndex::new(dag); + let mut runs = Vec::new(); + for group in groups { + let ids: Vec = group.views.iter().map(|v| v.node_id).collect(); + let Some(window) = group.refresh_window else { + return Err(IngestionError::Unsupported(format!( + "views {ids:?} are refreshed over an unbounded window" + ))); + }; + // The views of other groups it reads are view slots. + let in_group: HashSet = ids.iter().copied().collect(); + let read: BTreeSet = group + .views + .iter() + .flat_map(|v| v.inputs.materialized_views.iter().copied()) + .filter(|id| !in_group.contains(id)) + .collect(); + let read_views: Vec = views + .iter() + .filter(|v| read.contains(&v.node_id)) + .cloned() + .collect(); + let roots: Vec = ids.iter().map(|&id| id as u64).collect(); + let compiled = + compile(dag, &read_views, &roots).map_err(|error| IngestionError::Compile { + views: ids.clone(), + error, + })?; + let metrics = index.metrics_below(&ids, &read); + let margin_ms = options.view_retention_margin_ms; + let views = group + .views + .iter() + .map(|view| with_retention_margin(view, margin_ms)) + .collect::, _>>()?; + let retention_ms = views + .iter() + .map(|v| millis(v.retention_ms())) + .max() + .unwrap_or(0); + runs.push(RefreshGroupRun { + width_ms: millis(window.width_ms), + phase_ms: millis(window.phase_ms), + views, + compiled, + metrics, + retention_ms, + refreshed_through: None, + }); + } + // The `Scan`s the query run reads: below the roots, outside views. + let view_ids: HashSet = views.iter().map(|v| v.node_id).collect(); + let mut query_time_lookback_ms: HashMap> = HashMap::new(); + for scan in index.below(&dag.roots, &view_ids) { + let node = index.nodes[&scan]; + if let Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric }, + .. + }) = &node.payload + { + let margin_ms = millis(options.view_retention_margin_ms); + let lookback = node + .retention_ms + .map(|retention| millis(retention).saturating_add(margin_ms)); + let entry = query_time_lookback_ms + .entry(metric.clone()) + .or_insert(Some(0)); + *entry = match (*entry, lookback) { + (Some(a), Some(b)) => Some(a.max(b)), + _ => None, + }; + } + } + Ok(Self { + groups: runs, + query_time_lookback_ms, + options, + limits, + newest_sample_ms: None, + newest_clock_ms: None, + barrier_ms: None, + stats: IngestionStats::default(), + }) + } + + /// The watermark: no sample older than this is expected any more. The + /// newest routed sample, capped at the wall clock, minus the allowed + /// lateness, or the newest barrier if that is later. + pub fn watermark_ms(&self) -> Option { + let newest = match self.newest_clock_ms { + Some(clock) => self.newest_sample_ms.map(|newest| newest.min(clock)), + None => self.newest_sample_ms, + }; + let lateness = millis(self.options.allowed_lateness_ms); + let from_samples = newest.map(|newest| newest.saturating_sub(lateness)); + from_samples.max(self.barrier_ms) + } + + /// [`Self::ingest_at`] the current wall-clock time. + pub fn ingest(&mut self, store: &mut TimeIndexedStore, samples: Vec) { + self.ingest_at(store, samples, wall_clock_ms()); + } + + /// Stores `samples`, then refreshes every window the watermark has + /// passed. `now_ms` bounds sample times from above. + pub fn ingest_at( + &mut self, + store: &mut TimeIndexedStore, + samples: Vec, + now_ms: i64, + ) { + self.newest_clock_ms = self.newest_clock_ms.max(Some(now_ms)); + let latest_allowed_ms = now_ms.saturating_add(millis(self.options.max_future_ms)); + let mut oldest_kept: HashMap> = HashMap::new(); + for sample in samples { + let metric = sample.labels.metric.clone(); + if !store.raw_data().reads(&metric) { + self.stats.samples_unrouted += 1; + continue; + } + if sample.timestamp_ms > latest_allowed_ms { + self.stats.samples_dropped_future += 1; + continue; + } + self.start_groups(&metric, sample.timestamp_ms); + let cutoff = *oldest_kept + .entry(metric.to_string()) + .or_insert_with(|| self.oldest_kept_ms(&metric)); + if cutoff.is_some_and(|cutoff| sample.timestamp_ms <= cutoff) { + self.stats.samples_dropped_late += 1; + continue; + } + self.newest_sample_ms = self.newest_sample_ms.max(Some(sample.timestamp_ms)); + store.raw_data_mut().push(sample); + self.stats.samples_accepted += 1; + } + self.refresh_closed_windows(store); + } + + /// [`Self::advance_watermark_at`] the current wall-clock time. + pub fn advance_watermark(&mut self, store: &mut TimeIndexedStore, barrier_ms: i64) -> i64 { + self.advance_watermark_at(store, barrier_ms, wall_clock_ms()) + } + + /// An event-time barrier: the producer states that every sample at or + /// before `barrier_ms` has been written, so every window ending at or + /// before it closes without a later sample. Capped at `now_ms`; a + /// repeated or lower barrier does nothing. Returns the barrier in force. + pub fn advance_watermark_at( + &mut self, + store: &mut TimeIndexedStore, + barrier_ms: i64, + now_ms: i64, + ) -> i64 { + self.newest_clock_ms = self.newest_clock_ms.max(Some(now_ms)); + let barrier_ms = barrier_ms.min(now_ms); + if let Some(current) = self.barrier_ms.filter(|current| *current >= barrier_ms) { + return current; + } + self.barrier_ms = Some(barrier_ms); + self.refresh_closed_windows(store); + barrier_ms + } + + /// Starts each group scanning `metric` at its first sample: its first + /// window is skipped unless it starts at or after the declared stream + /// start. + fn start_groups(&mut self, metric: &str, timestamp_ms: i64) { + for group in &mut self.groups { + if group.refreshed_through.is_some() || !group.metrics.contains(metric) { + continue; + } + let first_end = group.window_end(timestamp_ms); + group.refreshed_through = Some(match self.options.stream_start_ms { + // The first window starting at or after the stream start. + Some(start) => { + group.window_end(start.saturating_add(group.width_ms)) - group.width_ms + } + None => { + self.stats.windows_skipped_first += 1; + first_end + } + }); + } + } + + /// Samples of `metric` at or before this are no longer read: older than + /// every window still to be refreshed and than what a query-time `Scan` + /// reads back from the watermark (less the allowed lateness, for + /// evaluations up to the newest sample). `None` while all are kept. + fn oldest_kept_ms(&self, metric: &str) -> Option { + let mut cutoff: Option = None; + let mut lower = |bound: Option| { + cutoff = Some(match (cutoff, bound) { + (_, None) => return false, + (None, Some(b)) => b, + (Some(a), Some(b)) => a.min(b), + }); + true + }; + for group in self.groups.iter().filter(|g| g.metrics.contains(metric)) { + // A group not started yet needs every sample. + if !lower(group.refreshed_through) { + return None; + } + } + if let Some(lookback) = self.query_time_lookback_ms.get(metric) { + let watermark = self.watermark_ms()?; + let lateness = millis(self.options.allowed_lateness_ms); + if !lower(lookback.map(|l| watermark.saturating_sub(l).saturating_sub(lateness))) { + return None; + } + } + cutoff + } + + fn refresh_closed_windows(&mut self, store: &mut TimeIndexedStore) { + let Some(watermark) = self.watermark_ms() else { + return; + }; + for group in &mut self.groups { + let Some(mut through) = group.refreshed_through else { + continue; + }; + // A window ending at or before this would already be dropped. + let expired_through = + group.window_end(watermark.saturating_sub(group.retention_ms)) - group.width_ms; + if through < expired_through { + self.stats.windows_skipped_expired += + ((expired_through - through) / group.width_ms) as u64; + through = expired_through; + } + while let Some(end) = through + .checked_add(group.width_ms) + .filter(|e| *e <= watermark) + { + match refresh(group, end, store, &self.limits) { + Ok(()) => self.stats.windows_refreshed += 1, + Err(error) => { + tracing::error!(views = ?group.views.iter().map(|v| v.node_id).collect::>(), end, %error, "ingestion run failed"); + self.stats.windows_failed += 1; + } + } + through = end; + } + group.refreshed_through = Some(through); + } + let metrics: Vec = self + .groups + .iter() + .flat_map(|g| g.metrics.iter().cloned()) + .chain(self.query_time_lookback_ms.keys().cloned()) + .collect::>() + .into_iter() + .collect(); + for metric in metrics { + if let Some(cutoff) = self.oldest_kept_ms(&metric) { + store.raw_data_mut().evict_through(&metric, cutoff); + } + } + } +} + +/// Runs `group`'s views in one ingestion run over `(end - W, end]` and +/// appends each view's rows to `store`. +fn refresh( + group: &RefreshGroupRun, + end: i64, + store: &mut TimeIndexedStore, + limits: &Limits, +) -> Result<(), asap_executor::Error> { + let start = end - group.width_ms; + let outputs = { + let instance = group.compiled.instantiate(store.run_store())?; + let context = RunContext::new( + Scope::Ingestion { + window_start_ms: start, + window_end_ms: end, + }, + limits.clone(), + )?; + let outputs = block_on(instance.execute_ingestion(group.compiled.roots(), context)?)?; + outputs + }; + let by_root: BTreeMap> = group + .compiled + .roots() + .iter() + .copied() + .zip(outputs) + .collect(); + for view in &group.views { + let batches = by_root + .get(&(view.node_id as u64)) + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .unwrap_or_default(); + store.append_view_rows(view, (start, end), batches)?; + } + Ok(()) +} + +/// The DAG's nodes by id and each node's inputs. +struct DagIndex<'a> { + nodes: HashMap< + PhysicalASAPNodeId, + &'a asap_executor::planner::plan::physical_asap_dag::PhysicalASAPDAGNode, + >, + inputs: HashMap>, +} + +impl<'a> DagIndex<'a> { + fn new(dag: &'a PhysicalASAPDAG) -> Self { + let mut inputs: HashMap> = HashMap::new(); + for edge in &dag.edges { + inputs.entry(edge.consumer).or_default().push(edge.producer); + } + Self { + nodes: dag.nodes.iter().map(|n| (n.id, n)).collect(), + inputs, + } + } + + /// The nodes reachable down from `from`, not passing a node of + /// `stop_at` (other than `from` itself). + fn below( + &self, + from: &[PhysicalASAPNodeId], + stop_at: &HashSet, + ) -> BTreeSet { + let mut seen = BTreeSet::new(); + let mut pending: Vec = from.to_vec(); + let starts: HashSet = from.iter().copied().collect(); + while let Some(id) = pending.pop() { + if (stop_at.contains(&id) && !starts.contains(&id)) || !seen.insert(id) { + continue; + } + pending.extend(self.inputs.get(&id).into_iter().flatten().copied()); + } + seen + } + + /// The metrics scanned below `views`, not reading through `read_views`. + fn metrics_below( + &self, + views: &[PhysicalASAPNodeId], + read_views: &BTreeSet, + ) -> BTreeSet { + let stop: HashSet = read_views.iter().copied().collect(); + crate::store::scanned_metrics(self.below(views, &stop).iter().map(|id| self.nodes[id])) + } +} + +/// `view` with its `retention_ms` extended by `margin_ms`, for the store +/// to keep its rows that much longer. The retention is not settable on a +/// `MaterializedView`, so the copy goes through its plan-file form. +fn with_retention_margin( + view: &MaterializedView, + margin_ms: u64, +) -> Result { + let error = |e: serde_json::Error| { + IngestionError::Unsupported(format!("materialized view {}: {e}", view.node_id)) + }; + let mut value = serde_json::to_value(view).map_err(error)?; + value["retention_ms"] = (view.retention_ms().saturating_add(margin_ms)).into(); + serde_json::from_value(value).map_err(error) +} + +fn millis(value: u64) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} + +/// Milliseconds since the Unix epoch. +pub(crate) fn wall_clock_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)) +} + +/// End of the tumbling window of width `width_ms` (on multiples of it) +/// that holds `timestamp_ms`. Windows are `(end - width_ms, end]`, like +/// PromQL ranges: a sample at a boundary belongs to the window it ends. +pub fn window_end(timestamp_ms: i64, width_ms: i64) -> i64 { + -((-timestamp_ms).div_euclid(width_ms)) * width_ms +} diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 11f8796a..48621d54 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -1,22 +1,21 @@ //! Backend data plane: receives Prometheus Remote Write samples and serves //! the Prometheus query API for an installed plan. //! -//! The installed plan is ASAPPlanner's selected `PhysicalASAPDAG`. The -//! engine thread builds its ingestion-time outputs pane by pane and answers -//! the plan's instant queries from them; every other query, and any planned -//! query it cannot answer, is forwarded to Prometheus. +//! The installed plan is ASAPPlanner's `InstallablePlan`: one selected +//! physical ASAP DAG. The engine thread stores the raw samples its `Scan`s +//! read, refreshes its materialized views in ingestion runs, and answers +//! the plan's instant queries with query runs, all through the Planner's +//! executor; every other query, and any planned query it cannot answer, is +//! forwarded to Prometheus (or refused with `--no-forward`). pub mod engine; -mod per_sample; -pub mod precompute; +pub mod ingestion; pub mod prometheus; pub mod query; -mod raw_sample_window; pub mod remote_write; -pub mod sds; pub mod series_ids; +pub mod store; -use std::collections::BTreeMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -28,9 +27,9 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::Router; use engine::{EngineHandle, EngineOptions, IngestError}; -use precompute::RawSample; use prometheus::InstantQuery; use query::Unanswered; +use store::RawSample; /// Response header naming what answered a query: `plan` or `forwarded`, or /// `refused` when `--no-forward` refused to forward it. @@ -58,17 +57,16 @@ pub struct DataPlane { } impl DataPlane { - /// Installs the plan serialized in `plan_json` (a - /// `asap_planner::InstallablePlan`) with [`EngineOptions::default`], except - /// that panes close `allowed_lateness_ms` after the newest sample passes - /// them. + /// Installs the plan file `plan_json` (an `asap_planner::InstallablePlan`) + /// with [`EngineOptions::default`], except that windows close + /// `allowed_lateness_ms` after the newest sample passes them. pub fn new( plan_json: String, prometheus_url: String, allowed_lateness_ms: u64, ) -> anyhow::Result { let mut options = EngineOptions::default(); - options.precompute.allowed_lateness_ms = allowed_lateness_ms; + options.ingestion.allowed_lateness_ms = allowed_lateness_ms; Self::with_options(plan_json, prometheus_url, options) } @@ -169,11 +167,11 @@ async fn write(State(state): State>, body: Bytes) -> Response { /// `POST /api/v1/precompute/watermark?time_ms=T`: the producer states that /// every sample at or before `T` has been written. Once the engine has run -/// every write acknowledged before this request, every pane with -/// `end <= T` is closed, so the last pane of a series that stopped sending -/// is built without a later sample. `T` is capped at the wall clock; a +/// every write acknowledged before this request, every window with +/// `end <= T` is closed, so the last window of a series that stopped sending +/// is refreshed without a later sample. `T` is capped at the wall clock; a /// repeated or lower `T` does nothing. A sample that arrives later for a -/// pane the barrier closed counts as late. Input stays open. With several +/// window the barrier closed counts as late. Input stays open. With several /// producers, call it only after all of them have written through `T`. /// Answers `{"barrier_ms": B}`, the barrier in force (`T` capped at the wall /// clock, or a higher earlier barrier); 400 `bad_data` with the reason when @@ -225,14 +223,13 @@ async fn ready(State(state): State>) -> StatusCode { } } -/// The data plane's counters, as JSON: the precompute engine's sample and -/// pane counters, the samples queued for it, and whether it runs. +/// The data plane's counters, as JSON: ingestion's sample and window +/// counters, the samples queued for it, and whether it runs. async fn status(State(state): State>) -> Response { let body = serde_json::json!({ "engine_running": state.engine.is_running(), "samples_queued": state.engine.queued_samples(), - "precompute": state.engine.stats(), - "query_run": state.engine.query_run_stats(), + "ingestion": state.engine.stats(), }); ( [(header::CONTENT_TYPE, "application/json")], @@ -276,7 +273,7 @@ async fn query( return (StatusCode::BAD_REQUEST, body).into_response(); } }; - let reason = match answer(&state, instant, &headers).await { + let reason = match answer(&state, instant).await { Ok(response) => { let mut response = axum::Json(response).into_response(); response @@ -298,38 +295,10 @@ async fn query( response } -/// Runs a planned instant query. Bound raw-sample windows are read from Prometheus with -/// the caller's end-to-end `headers`, as a forwarded query would be. -async fn answer( - state: &DataPlane, - query: InstantQuery, - headers: &HeaderMap, -) -> Result { - let time_ms = query.time_ms.unwrap_or_else(|| { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_millis() as i64) - }); - let prepared = state - .engine - .bound_raw_sample_windows(query.query, time_ms) - .await?; - let ranges = &prepared.windows; - let mut fetch_headers = forwarded_headers(headers); - // The raw read is a GET without the caller's body. - fetch_headers.remove(header::CONTENT_TYPE); - // The bound raw-sample windows are fetched concurrently. - let fetches = ranges.iter().map(|range| async { - prometheus::fetch(&state.client, &state.prometheus_url, &fetch_headers, range) - .await - .map(|samples| (range.node, samples)) - }); - let raw: BTreeMap> = futures::future::try_join_all(fetches) - .await - .map_err(|e| Unanswered::Execution(format!("fetching raw samples: {e}")))? - .into_iter() - .collect(); - let samples = state.engine.evaluate(prepared, time_ms, raw).await?; +/// Runs a planned instant query at its time, or now. +async fn answer(state: &DataPlane, query: InstantQuery) -> Result { + let time_ms = query.time_ms.unwrap_or_else(ingestion::wall_clock_ms); + let samples = state.engine.evaluate(query.query, time_ms).await?; Ok(prometheus::vector_response(time_ms, &samples)) } diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 8053ffc7..dd0ea1a4 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -4,7 +4,7 @@ use std::time::Duration; use clap::Parser; use data_plane::engine::EngineOptions; -use data_plane::precompute::PrecomputeOptions; +use data_plane::ingestion::IngestionOptions; use data_plane::{router, DataPlane, DEFAULT_FORWARD_TIMEOUT}; /// Serve Remote Write ingestion and the Prometheus query API for a plan. @@ -21,27 +21,34 @@ struct Args { forward_timeout_seconds: u64, #[arg(long, default_value = "0.0.0.0:8088")] listen: String, - /// How long after the newest sample passes a pane's end it is built. - #[arg(long, default_value_t = PrecomputeOptions::default().allowed_lateness_ms)] + /// How long after the newest sample passes a window's end it is refreshed. + #[arg(long, default_value_t = IngestionOptions::default().allowed_lateness_ms)] allowed_lateness_ms: u64, /// Samples further ahead of the wall clock than this are dropped. - #[arg(long, default_value_t = PrecomputeOptions::default().max_future_ms)] + #[arg(long, default_value_t = IngestionOptions::default().max_future_ms)] max_future_ms: u64, /// Where the producers' stream starts (ms since the epoch): every raw - /// sample at or after it is written to the data plane, so a first pane - /// that starts at or after it is complete and is built. By default the - /// first pane each definition sees is skipped. + /// sample at or after it is written to the data plane, so a first window + /// that starts at or after it is complete and is refreshed. By default + /// the first window each refresh group sees is skipped. #[arg(long)] stream_start_ms: Option, + /// How much longer than the plan's retention views and raw samples are + /// kept, so a query this far behind the newest refreshed window is still + /// answered. + #[arg(long, default_value_t = IngestionOptions::default().view_retention_margin_ms)] + view_retention_margin_ms: u64, /// Samples queued for the engine beyond which Remote Write gets 503. #[arg(long, default_value_t = EngineOptions::default().max_queued_samples)] max_queued_samples: u64, /// Never forward a query to Prometheus: refuse a plan with a query the /// data plane cannot run, and answer any query the plan does not answer - /// with an error giving the reason. Raw samples are still read from - /// Prometheus. + /// with an error giving the reason. #[arg(long)] no_forward: bool, + /// The most bytes one ingestion or query run may hold. + #[arg(long, default_value_t = EngineOptions::default().run_memory_limit_bytes)] + run_memory_limit_bytes: usize, } #[tokio::main] @@ -56,13 +63,15 @@ async fn main() -> anyhow::Result<()> { plan, args.prometheus_url, EngineOptions { - precompute: PrecomputeOptions { + ingestion: IngestionOptions { allowed_lateness_ms: args.allowed_lateness_ms, max_future_ms: args.max_future_ms, stream_start_ms: args.stream_start_ms, + view_retention_margin_ms: args.view_retention_margin_ms, }, max_queued_samples: args.max_queued_samples, no_forward: args.no_forward, + run_memory_limit_bytes: args.run_memory_limit_bytes, }, )? .with_forward_timeout(Duration::from_secs(args.forward_timeout_seconds)), diff --git a/data_plane/src/per_sample.rs b/data_plane/src/per_sample.rs deleted file mode 100644 index 4a6fe764..00000000 --- a/data_plane/src/per_sample.rs +++ /dev/null @@ -1,435 +0,0 @@ -//! Per-sample ingestion: a definition's build, `Scan -> TimeShift* -> -//! TimeRange(pane) -> SummaryAgg`, run one sample at a time, in columns. -//! -//! - **Per series, once:** the stateless prefix. The window's -//! [`RawSampleWindowBatcher`] applies the `Scan`'s label matchers to the series and -//! caches its text cells; the cells of the `SummaryAgg`'s group columns -//! are its group. -//! - **Per sample:** the executor's `StateColumnUpdater` for the open pane -//! takes the value and time into the group's state in place, the code the -//! batch `SummaryAgg` runs per row. -//! - **At pane close:** the states are sealed into a state column and the -//! pane is built from columns (`Batch::try_from_columns`): the group -//! columns' texts, the pane's timestamp, and the states. -//! -//! A group is one series (the group columns include the series identity), -//! so a counter's samples reach its state in the time order the batch build -//! requires of one series. -//! -//! A build of another shape keeps the batch build over the pane's samples: -//! a row filter, a keyed or shared-grid family, an operator between the -//! `TimeRange` and the `SummaryAgg`, matchers on the sample's value or -//! time, or groups that are not series. The -//! Planner places no such build at ingestion time for the PromQL examples. - -use std::collections::BTreeMap; -use std::sync::{Arc, OnceLock}; - -use asap_executor::arrow::array::{Array, TimestampMillisecondArray, UInt32Array}; -use asap_executor::planner::ir::operator::Reduction; -use asap_executor::planner::ir::physical_export::PhysicalASAPDAG; -use asap_executor::planner::ir::scalar::ColumnRef; -use asap_executor::planner::ir::schema::{ - DataType, FieldDataType, GroupingStrategy, SummaryInputExpr, -}; -use asap_executor::planner::ir::{ASAPOp, NonASAPOp, Operator}; -use asap_executor::runtime::Scope; -use asap_executor::values::{Batch, BatchColumn, SchemaRef}; -use asap_executor::StateColumnUpdater; - -use crate::precompute::{RawSample, SummaryBuild}; -use crate::raw_sample_window::{ - is_stale, text_column, RawSampleWindowBatcher, SeriesCells, SERIES_IDENTITY_COLUMN, -}; - -/// A group: its texts in the `SummaryAgg`'s group columns, in their order. -type GroupKey = Arc<[Arc]>; - -/// How a definition is built per sample; compiled once per definition. -pub(crate) struct PerSampleSpec { - family: FieldDataType, - /// The input group columns, in the `SummaryAgg`'s order. - groups: Vec, - output: SchemaRef, - /// Output column of the states, and of the pane's timestamp. - state_column: usize, - time_column: usize, - /// Per output text column: its position in a group's key. - output_group_positions: Vec<(usize, usize)>, - /// How much the output timestamp is after the pane's end, found by - /// running the plan's build once. - time_offset_ms: OnceLock, -} - -/// One series as a per-sample build has seen it. -struct SeriesEntry { - /// Its group, or `None` when the `Scan` does not select it. - key: Option, - /// Its newest sample time: a series unseen for longer than the - /// definition's retention is forgotten. - newest_ms: i64, - /// The open pane it last took a sample in, and its group number there, - /// so a run of samples into one pane looks the series up once. - pane: Option<(i64, u32)>, -} - -/// One open pane: a summary state per series that took a sample in it. -struct OpenPane { - updater: StateColumnUpdater, - /// Each group's key and newest sample time, by its number in the pane. - groups: Vec<(GroupKey, i64)>, - /// Group numbers, by series id. - numbers: ahash::HashMap, -} - -/// What a per-sample build did with one sample. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Taken { - /// Taken into its series' state, or not a sample of any (a stale - /// marker, or a series the `Scan` does not select). - Taken, - /// Not after its series' previous sample in the pane (a resent sample, - /// for instance): dropped, not failing the pane or counted twice. - Duplicate, -} - -/// The open panes of a definition built per sample. -#[derive(Default)] -pub(crate) struct OpenPanes { - /// Each series seen and not forgotten, by series id. - series: ahash::HashMap, - /// By pane end: its states, or why it failed. - panes: BTreeMap>, -} - -impl OpenPanes { - /// Drops the open panes ending before `end_ms`, which are skipped. - pub(crate) fn drop_before(&mut self, end_ms: i64) { - self.panes = self.panes.split_off(&end_ms); - } - - /// The series remembered, and each open pane's number of states. - #[cfg(test)] - pub(crate) fn sizes(&self) -> (usize, Vec) { - let states = self - .panes - .values() - .map(|pane| pane.as_ref().map_or(0, |p| p.updater.group_count())) - .collect(); - (self.series.len(), states) - } -} - -impl PerSampleSpec { - /// The per-sample build of `pane`'s definition, or `None` when its shape - /// keeps the batch build. - pub(crate) fn new(dag: &PhysicalASAPDAG, pane: &SummaryBuild) -> Option { - let definition = &pane.definition; - let window = definition.window.as_ref()?; - let raw_input = window.time_range_node_id as u64; - let node = |id: u64| dag.nodes.iter().find(|n| n.id as u64 == id); - let builder = node(definition.builder)?; - let inputs: Vec = dag - .edges - .iter() - .filter(|e| e.consumer as u64 == definition.builder) - .map(|e| e.producer as u64) - .collect(); - if inputs != [raw_input] { - return None; - } - let Operator::ASAP(ASAPOp::SummaryAgg { - family, - input: update, - reduction, - grouping, - filter: None, - .. - }) = &builder.payload - else { - return None; - }; - if update.item.is_some() || *grouping != GroupingStrategy::PerSubpopulationInstance { - return None; - } - let input = &node(raw_input)?.output_schema; - let SummaryInputExpr::Column(column) = &update.weight else { - return None; - }; - let name = match column { - ColumnRef::SampleValue => "value", - ColumnRef::Named(name) => name.as_str(), - _ => return None, - }; - let value = input.fields.iter().position(|f| f.name == name)?; - let time = input.time_index?; - let groups: Vec = match reduction { - Reduction::PerEntity => (0..input.fields.len()) - .filter(|&c| c != value && c != time) - .collect(), - Reduction::Reduce(keys) if !keys.is_without() && !keys.keys().is_empty() => { - keys.keys().to_vec() - } - _ => return None, - }; - if groups - .iter() - .any(|&c| input.fields[c].dtype != FieldDataType::Plain(DataType::Utf8)) - { - return None; - } - // The cached prefix holds only if the matchers read labels alone. - let scan = node(window.scan_node_id as u64)?; - let Operator::NonASAP(NonASAPOp::Scan { predicates, .. }) = &scan.payload else { - return None; - }; - let read = serde_json::to_value(predicates).ok()?; - if columns_read(&read).any(|c| c == value || c == time) { - return None; - } - StateColumnUpdater::new(family).ok()?; - let output = Arc::new(definition.output_schema.clone()); - let state_column = output - .fields - .iter() - .position(|f| !matches!(f.dtype, FieldDataType::Plain(_)))?; - let time_column = output.time_index?; - // Every other output column repeats a group column of its name. - let output_group_positions = output - .fields - .iter() - .enumerate() - .filter(|(i, _)| *i != state_column && *i != time_column) - .map(|(i, field)| { - let position = groups - .iter() - .position(|&c| input.fields[c].name == field.name)?; - (field.dtype == FieldDataType::Plain(DataType::Utf8)).then_some((i, position)) - }) - .collect::>>()?; - // A group is one series. - if !groups - .iter() - .any(|&c| input.fields[c].name == SERIES_IDENTITY_COLUMN) - { - return None; - } - Some(Self { - family: family.clone(), - groups, - output, - state_column, - time_column, - output_group_positions, - time_offset_ms: OnceLock::new(), - }) - } - - /// Takes `sample` into the pane ending at `end_ms`, open in `open`: - /// its series' summary state there is updated in place. `window` gives - /// a series' group the first time it is seen. - pub(crate) fn update( - &self, - window: &RawSampleWindowBatcher, - open: &mut OpenPanes, - sample: &RawSample, - end_ms: i64, - ) -> Taken { - // A stale marker ends a series; it is not a sample of its state. - if is_stale(sample.value) { - return Taken::Taken; - } - let id = sample.labels.id; - let entry = open.series.entry(id).or_insert_with(|| SeriesEntry { - key: window.cells(sample).map(|cells| { - self.groups - .iter() - .map(|&c| cells.texts[c].clone().unwrap_or_else(|| Arc::from(""))) - .collect() - }), - newest_ms: sample.timestamp_ms, - pane: None, - }); - entry.newest_ms = entry.newest_ms.max(sample.timestamp_ms); - let Some(key) = &entry.key else { - return Taken::Taken; - }; - let pane = open.panes.entry(end_ms).or_insert_with(|| { - StateColumnUpdater::new(&self.family) - .map(|updater| OpenPane { - updater, - groups: Vec::new(), - numbers: ahash::HashMap::default(), - }) - .map_err(|e| e.to_string()) - }); - let Ok(pane) = pane else { - return Taken::Taken; - }; - let number = match entry.pane { - Some((end, number)) if end == end_ms => Ok(number), - _ => match pane.numbers.get(&id) { - Some(&number) => Ok(number), - None => pane.updater.add_group().inspect(|&number| { - pane.numbers.insert(id, number); - pane.groups.push((key.clone(), i64::MIN)); - }), - }, - }; - let result = number.and_then(|number| { - entry.pane = Some((end_ms, number)); - let newest = &mut pane.groups[number as usize].1; - // As Prometheus does, a series' sample not after its previous one - // (a resent sample, for instance) is dropped: a counter's state - // refuses it, and any other state would count it twice. - if sample.timestamp_ms <= *newest { - return Ok(Taken::Duplicate); - } - *newest = sample.timestamp_ms; - pane.updater - .update(number, sample.value, sample.timestamp_ms) - .map(|()| Taken::Taken) - }); - match result { - Ok(taken) => taken, - Err(error) => { - open.panes.insert(end_ms, Err(error.to_string())); - Taken::Taken - } - } - } - - /// The pane ending at `end_ms`, which closes: a row per group (series) - /// that took a sample. Then the series not seen for longer than - /// `keep_ms` before the pane's end are forgotten. - pub(crate) fn close( - &self, - build: &SummaryBuild, - open: &mut OpenPanes, - end_ms: i64, - keep_ms: i64, - scope: &Scope, - ) -> Result { - let oldest_ms = end_ms.saturating_sub(keep_ms); - open.series.retain(|_, entry| entry.newest_ms > oldest_ms); - build.window.forget_unseen(oldest_ms); - let (updater, groups) = match open.panes.remove(&end_ms) { - Some(pane) => { - let pane = pane?; - (pane.updater, pane.groups) - } - None => ( - StateColumnUpdater::new(&self.family).map_err(|e| e.to_string())?, - Vec::new(), - ), - }; - self.pane_batch(build, updater, &groups, end_ms, scope) - } - - /// The pane's batch: a row per group, in group key order as the batch - /// build emits them, with its texts, the pane's timestamp and its state. - fn pane_batch( - &self, - build: &SummaryBuild, - updater: StateColumnUpdater, - groups: &[(GroupKey, i64)], - end_ms: i64, - scope: &Scope, - ) -> Result { - let mut rows: Vec<(&GroupKey, u32)> = groups - .iter() - .enumerate() - .map(|(number, (key, _))| (key, number as u32)) - .collect(); - rows.sort_unstable_by(|a, b| a.0.cmp(b.0)); - // Found once; a pane fails, and the next one tries again, until the - // build gives a timestamp. - let offset = match self.time_offset_ms.get() { - Some(offset) => *offset, - None => { - let offset = self.calibrate(build, end_ms, scope)?; - *self.time_offset_ms.get_or_init(|| offset) - } - }; - let states = Arc::new(updater.seal()); - let columns = (0..self.output.fields.len()) - .map(|column| { - if column == self.state_column { - return Ok(BatchColumn::States { - indices: rows - .iter() - .map(|(_, number)| *number) - .collect::(), - states: states.clone(), - }); - } - if column == self.time_column { - return Ok(BatchColumn::Plain(Arc::new( - TimestampMillisecondArray::from_iter_values( - rows.iter().map(|_| end_ms + offset), - ), - ))); - } - let (_, position) = self - .output_group_positions - .iter() - .find(|(output, _)| *output == column) - .expect("every other column repeats a group column"); - text_column(rows.iter().map(|(key, _)| key[*position].as_ref())) - .map(BatchColumn::Plain) - }) - .collect::, asap_executor::Error>>() - .map_err(|e| e.to_string())?; - Batch::try_from_columns(self.output.clone(), columns).map_err(|e| e.to_string()) - } - - /// How much the build's output timestamp is after the pane's end: the - /// plan's build run once over one row. - fn calibrate(&self, pane: &SummaryBuild, end_ms: i64, scope: &Scope) -> Result { - let texts = pane - .window - .schema - .fields - .iter() - .map(|field| field.dtype == FieldDataType::Plain(DataType::Utf8)) - .map(|text| text.then(|| Arc::from(""))) - .collect(); - let cells = Arc::new(SeriesCells { texts }); - let row = pane - .window - .batch_of(&[(cells, end_ms, 1.0)]) - .map_err(|e| e.to_string())?; - let built = pane - .run_batch(row, scope.clone()) - .map_err(|e| e.to_string())?; - let time = built - .batches - .iter() - .find(|batch| batch.num_rows() > 0) - .and_then(|batch| { - batch - .column(self.time_column) - .as_any() - .downcast_ref::() - .filter(|times| !times.is_null(0)) - .map(|times| times.value(0)) - }) - .ok_or("the build gave no timestamp")?; - Ok(time - end_ms) - } -} - -/// The column indices an expression's JSON form reads (`{"Column": i}`). -fn columns_read(expression: &serde_json::Value) -> Box + '_> { - match expression { - serde_json::Value::Object(fields) => Box::new(fields.iter().flat_map(|(name, value)| { - let column = match (name.as_str(), value) { - ("Column", serde_json::Value::Number(n)) => n.as_u64().map(|c| c as usize), - _ => None, - }; - column.into_iter().chain(columns_read(value)) - })), - serde_json::Value::Array(items) => Box::new(items.iter().flat_map(columns_read)), - _ => Box::new(std::iter::empty()), - } -} diff --git a/data_plane/src/precompute.rs b/data_plane/src/precompute.rs deleted file mode 100644 index 69cac61e..00000000 --- a/data_plane/src/precompute.rs +++ /dev/null @@ -1,694 +0,0 @@ -//! Precompute engine: builds each stored summary pane once, when the -//! watermark passes its end. -//! -//! A pane `(end - pane, end]` of a [`SummaryDefinition`] closes when the -//! watermark (the newest sample time minus the allowed lateness) reaches -//! `end`. Where the definition's shape allows ([`crate::per_sample`]), each -//! sample of its `Scan`'s metric updates its series' summary state in the -//! open pane in place, and closing the pane seals the states into one batch -//! built from columns. Otherwise the engine buffers the pane's samples, and -//! closing it selects the pane's rows with the `Scan`'s label matchers and -//! runs the definition's build from its `TimeRange` input up to its output, -//! under ingestion scope. The output is stored and never changes; samples -//! older than a closed pane are dropped. The first pane a definition sees -//! may have started before the data plane did, so it is never built: -//! queries that need it are not answered from stored panes. A producer that -//! declares where its stream starts (`stream_start_ms`) makes a first pane -//! starting at or after that point complete, and it is built. -//! -//! Only samples of a metric some definition reads move the watermark, and a -//! sample further ahead of the wall clock than `max_future_ms` is dropped, so -//! one bad timestamp cannot close every pane up to it. When the watermark -//! jumps, panes that would already be evicted are skipped, not built. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use asap_executor::operators::Operator; -use asap_executor::physical_planner::{compile, CompiledPhysicalDAG, InputContract, Source}; -use asap_executor::planner::ir::physical_export::PhysicalASAPDAG; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::values::{Batch, SchemaRef}; -use asap_executor::RawSampleSelector; -use futures::executor::block_on; -use futures::StreamExt; - -use asap_planner::StoredOutputBuild; - -use crate::per_sample::{OpenPanes as OpenStates, PerSampleSpec, Taken}; -use crate::raw_sample_window::RawSampleWindowBatcher; -use crate::sds::{DefinitionId, Sds, SummaryDefinition, SummaryInstanceKey, SummaryStore}; -use crate::series_ids::Series; - -/// One raw sample: its series (interned: its id and full label set, -/// `__name__` included), time and value. -#[derive(Debug, Clone, PartialEq)] -pub struct RawSample { - pub labels: Arc, - pub timestamp_ms: i64, - pub value: f64, -} - -#[derive(Debug, thiserror::Error)] -pub enum PrecomputeError { - #[error("stored summaries: {0}")] - Plan(String), - #[error("stored summary {0:?}: {1}")] - Definition(DefinitionId, String), - #[error(transparent)] - Executor(#[from] asap_executor::Error), -} - -/// Counters for what happened to incoming samples, each sample counted once -/// (`samples_*`), and to panes, each pane of each definition counted once -/// (`panes_*`). -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize)] -pub struct PrecomputeStats { - /// Samples buffered for at least one pane. - pub samples_accepted: u64, - /// Samples of a pane already built or skipped, other than the first - /// pane after startup. - pub samples_dropped_late: u64, - /// Samples, of a stored output built per sample, whose time is not - /// after their series' previous sample in the pane (a resent sample, - /// for instance): dropped, as Prometheus drops them. - pub samples_dropped_duplicate: u64, - /// Samples of the first pane after startup, which is never built. - pub samples_skipped_first_pane: u64, - /// Samples more than `max_future_ms` ahead of the wall clock. - pub samples_dropped_future: u64, - /// Samples of a metric no stored summary reads. - pub samples_unrouted: u64, - /// Samples of a metric only stored outputs built in a `QueryRun` read: they move the - /// watermark, which says when such a summary is closed, but are not - /// buffered (the query engine reads them from Prometheus). - pub samples_query_run_only: u64, - /// Panes built and stored. - pub panes_built: u64, - /// Panes whose build failed; they are skipped, not retried. - pub panes_failed: u64, - /// Panes not built because they would already be evicted when the - /// watermark reached them. - pub panes_skipped_expired: u64, -} - -/// How the precompute engine treats sample times. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PrecomputeOptions { - /// A pane closes when the newest routed sample is this far past its end. - pub allowed_lateness_ms: u64, - /// Samples further ahead of the wall clock than this are dropped. - pub max_future_ms: u64, - /// Where the producer's stream starts: every raw sample at or after it - /// reaches the data plane. A definition's first pane is then built when - /// it starts at or after it. `None`: the first pane may have started - /// before the data plane did, so it is never built. - pub stream_start_ms: Option, -} - -impl Default for PrecomputeOptions { - fn default() -> Self { - Self { - allowed_lateness_ms: 15_000, - max_future_ms: 5 * 60_000, - stream_start_ms: None, - } - } -} - -/// A summary instance as stored in the SDS: its rows, in the batches its -/// build produced (one, when built per sample). Batches share their -/// buffers when cloned, so binding a stored pane to a query copies nothing. -#[derive(Debug, Clone, Default)] -pub struct SummaryInstance { - pub batches: Vec, -} - -impl SummaryInstance { - pub fn num_rows(&self) -> usize { - self.batches.iter().map(Batch::num_rows).sum() - } - - /// Every row, for tests; the data plane reads rows only to encode an - /// answer. - pub fn to_rows(&self) -> Vec> { - self.batches.iter().flat_map(Batch::to_rows).collect() - } -} - -/// How one definition's summaries are built from raw samples with the -/// plan's own build. The precompute engine builds stored outputs built in -/// an `IngestionRun` with it where it does not build them per sample, and -/// the query engine `QueryRun` ones. -pub struct SummaryBuild { - pub definition: SummaryDefinition, - /// The definition's `RawSampleWindow`: a coverage's raw samples to the - /// build's input batch, column by column, with the `Scan`'s label - /// matchers. - pub(crate) window: RawSampleWindowBatcher, - /// The `TimeRange` node the window's batch feeds. - raw_input: u64, - build: CompiledPhysicalDAG, -} - -impl SummaryBuild { - /// Compiles `definition`'s build from its `TimeRange` up to its builder. - /// A definition built from other stored outputs is refused until the - /// engine can build it once its inputs are sealed. - pub fn new( - dag: &PhysicalASAPDAG, - definition: &SummaryDefinition, - ) -> Result { - let error = |message: String| PrecomputeError::Definition(definition.id.clone(), message); - if !definition.stored_inputs.is_empty() { - return Err(error( - "built from other stored outputs, which the data plane cannot build yet".into(), - )); - } - if definition.pane_ms <= 0 { - return Err(error(format!("pane width {} ms", definition.pane_ms))); - } - // `batch` is called with each coverage's end, so the window's - // offset from evaluation time is not used. - // A series is cached for as long as the definition's summaries are kept. - let keep_ms = definition.retention_ms.saturating_add(definition.pane_ms); - let Some(raw) = &definition.window else { - return Err(error("reads no raw-sample window".into())); - }; - let raw_input = raw.time_range_node_id as u64; - let window = RawSampleWindowBatcher::new(dag, raw, keep_ms)?; - let build = compile( - dag, - BTreeMap::from([(raw_input, InputContract::bounded(window.schema.clone()))]), - &[definition.builder], - )?; - Ok(Self { - definition: definition.clone(), - raw_input, - window, - build, - }) - } - - /// The metric whose samples the build reads. - pub fn metric(&self) -> &str { - self.window.metric() - } - - /// Selects the coverage ending at `end_ms` from `samples` with the plan's - /// `Scan` matchers, then runs the build over those rows under `scope`. - pub fn run( - &self, - end_ms: i64, - samples: &[RawSample], - scope: Scope, - ) -> Result { - self.run_batch(self.window.batch(end_ms, samples)?, scope) - } - - /// The build over `input`, a coverage's selected input batch. - pub(crate) fn run_batch( - &self, - input: Batch, - scope: Scope, - ) -> Result { - let input: Source<'_> = Box::new(Operator::source(input.schema().clone(), vec![input])?); - let build = self - .build - .instantiate(BTreeMap::from([(self.raw_input, input)]))?; - Ok(SummaryInstance { - batches: run(&build, self.build.roots(), scope)?, - }) - } -} - -struct Builder { - summary: SummaryBuild, - /// The build run per sample, unless its shape keeps the plan's build. - per_sample: Option, - /// For the plan's build: samples of open panes, by pane end. - open: BTreeMap>, - /// For the per-sample build: the open panes' summary states. - states: OpenStates, - /// End of the first pane seen, which is skipped. `None` until then. - first_pane_end: Option, - /// End of the newest pane built or skipped; samples in it or before it - /// are dropped. `None` until the first sample. - built_through: Option, -} - -/// What one definition did with one sample. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -enum Outcome { - Late, - Duplicate, - FirstPane, - Buffered, -} - -pub struct Precompute { - builders: Vec, - /// Metrics read by stored outputs built in a `QueryRun`, with each one's newest routed - /// sample time. - query_run_metrics: BTreeMap>, - allowed_lateness_ms: i64, - max_future_ms: i64, - stream_start_ms: Option, - newest_sample_ms: Option, - /// The newest wall-clock time an ingest ran at: the watermark never - /// passes it, so a series stamped ahead of the others cannot close - /// panes early. - newest_clock_ms: Option, - /// The newest event-time barrier a producer declared: every pane ending - /// at or before it is closed. - barrier_ms: Option, - pub store: SummaryStore, - pub stats: PrecomputeStats, -} - -impl Precompute { - /// Compiles one build per definition of `sds` built in an ingestion run, - /// in dependency order; `sds` describes `dag`. A definition is built per - /// sample where its shape allows ([`crate::per_sample`]), else with the - /// plan's build over each pane's samples. - pub fn new( - dag: PhysicalASAPDAG, - sds: &Sds, - options: PrecomputeOptions, - ) -> Result { - Self::with(&dag, sds, options, true) - } - - /// [`Self::new`] building every pane with the plan's own build over its - /// samples: the reference the per-sample builds equal. - pub fn batch( - dag: &PhysicalASAPDAG, - sds: &Sds, - options: PrecomputeOptions, - ) -> Result { - Self::with(dag, sds, options, false) - } - - /// How many definitions are built per sample. - pub fn per_sample_definitions(&self) -> usize { - self.builders - .iter() - .filter(|b| b.per_sample.is_some()) - .count() - } - - fn with( - dag: &PhysicalASAPDAG, - sds: &Sds, - options: PrecomputeOptions, - per_sample: bool, - ) -> Result { - let dag = dag.clone(); - let mut builders = Vec::new(); - // `IngestionRun` definitions in dependency order; `QueryRun` ones are - // the query engine's. - let order = sds - .ingestion_build_order() - .map_err(|e| PrecomputeError::Plan(e.to_string()))?; - for id in &order { - let summary = SummaryBuild::new(&dag, &sds.definitions[id])?; - let per_sample = per_sample - .then(|| PerSampleSpec::new(&dag, &summary)) - .flatten(); - tracing::info!(definition = ?id, per_sample = per_sample.is_some(), "pane build"); - builders.push(Builder { - summary, - per_sample, - open: BTreeMap::new(), - states: OpenStates::default(), - first_pane_end: None, - built_through: None, - }); - } - // Samples of a metric only stored outputs built in a `QueryRun` read move the - // watermark, which says when such a summary's coverage is closed. - // A `QueryRun` definition whose selector does not build is not - // tracked here; the query engine forwards the queries that read it. - let mut query_run_metrics = BTreeMap::new(); - for definition in sds.definitions.values() { - if definition.built_at == StoredOutputBuild::QueryRun { - let Some(window) = &definition.window else { - continue; - }; - match RawSampleSelector::new(&dag, window) { - Ok(selector) => { - query_run_metrics.insert(selector.metric().to_string(), None); - } - Err(error) => tracing::warn!( - definition = ?definition.id, %error, - "stored output built in a `QueryRun` cannot be built; its queries are forwarded" - ), - } - } - } - let millis = |value: u64| i64::try_from(value).unwrap_or(i64::MAX); - Ok(Self { - builders, - query_run_metrics, - allowed_lateness_ms: millis(options.allowed_lateness_ms), - max_future_ms: millis(options.max_future_ms), - stream_start_ms: options.stream_start_ms, - newest_sample_ms: None, - newest_clock_ms: None, - barrier_ms: None, - store: SummaryStore::new(), - stats: PrecomputeStats::default(), - }) - } - - /// What ingestion has seen, for the query engine's store decisions: the - /// watermark and the newest sample of each metric a `QueryRun` stored - /// output reads. - pub fn progress(&self) -> crate::query::IngestionProgress { - crate::query::IngestionProgress { - watermark_ms: self.watermark_ms(), - newest_sample_ms: self - .query_run_metrics - .iter() - .filter_map(|(metric, newest)| Some((metric.clone(), (*newest)?))) - .collect(), - } - } - - /// The watermark: no sample older than this is expected any more. It is - /// the newest routed sample, capped at the wall clock, minus the allowed - /// lateness, or the newest barrier if that is later. - pub fn watermark_ms(&self) -> Option { - let newest = match self.newest_clock_ms { - Some(clock) => self.newest_sample_ms.map(|newest| newest.min(clock)), - None => self.newest_sample_ms, - }; - let from_samples = newest.map(|newest| newest.saturating_sub(self.allowed_lateness_ms)); - from_samples.max(self.barrier_ms) - } - - /// [`Self::advance_watermark_at`] the current wall-clock time. - pub fn advance_watermark(&mut self, barrier_ms: i64) -> i64 { - self.advance_watermark_at(barrier_ms, wall_clock_ms()) - } - - /// An event-time barrier: the producer states that every sample at or - /// before `barrier_ms` has been written. Closes every pane with - /// `end <= barrier_ms` (panes are `(end - width, end]`), so a pane whose - /// series stopped sending closes without a later sample. The barrier is - /// capped at `now_ms`; a repeated or lower barrier does nothing. A sample - /// that arrives later for a pane the barrier closed counts as late. - /// Returns the barrier in force. - pub fn advance_watermark_at(&mut self, barrier_ms: i64, now_ms: i64) -> i64 { - self.newest_clock_ms = self.newest_clock_ms.max(Some(now_ms)); - let barrier_ms = barrier_ms.min(now_ms); - if self.barrier_ms.is_some_and(|current| current >= barrier_ms) { - return self.barrier_ms.unwrap_or(barrier_ms); - } - self.barrier_ms = Some(barrier_ms); - self.close_panes(); - barrier_ms - } - - /// [`Self::ingest_at`] the current wall-clock time. - pub fn ingest(&mut self, samples: &[RawSample]) { - self.ingest_at(samples, wall_clock_ms()); - } - - /// Buffers `samples` into their panes, then builds every pane the - /// watermark has passed. `now_ms` is the wall-clock time that bounds - /// sample times from above. - pub fn ingest_at(&mut self, samples: &[RawSample], now_ms: i64) { - self.newest_clock_ms = self.newest_clock_ms.max(Some(now_ms)); - let latest_allowed_ms = now_ms.saturating_add(self.max_future_ms); - for sample in samples { - let metric = Some(&*sample.labels.metric).filter(|m| !m.is_empty()); - let query_run = metric.is_some_and(|m| self.query_run_metrics.contains_key(m)); - if !query_run - && !self - .builders - .iter() - .any(|b| metric == Some(b.summary.metric())) - { - self.stats.samples_unrouted += 1; - continue; - } - if sample.timestamp_ms > latest_allowed_ms { - self.stats.samples_dropped_future += 1; - continue; - } - let mut outcome: Option = None; - for builder in &mut self.builders { - if metric != Some(builder.summary.metric()) { - continue; - } - let this = match pane_end(sample.timestamp_ms, builder.summary.definition.pane_ms) { - None => Outcome::Late, - Some(end) => { - let pane_ms = builder.summary.definition.pane_ms; - // A first pane that starts at or after the declared - // stream start is complete: nothing is skipped, and - // the panes from it on are built. - let complete = self - .stream_start_ms - .is_some_and(|start| end - pane_ms >= start); - let skipped = if complete { end - pane_ms } else { end }; - let first = *builder.first_pane_end.get_or_insert(skipped); - let built = *builder.built_through.get_or_insert(skipped); - if end == first { - Outcome::FirstPane - } else if end <= built { - Outcome::Late - } else { - match &builder.per_sample { - // A series the Scan does not select counts - // as taken, as the plan's build takes and - // then filters it. - Some(spec) => { - let window = &builder.summary.window; - match spec.update(window, &mut builder.states, sample, end) { - Taken::Taken => Outcome::Buffered, - Taken::Duplicate => Outcome::Duplicate, - } - } - None => { - builder.open.entry(end).or_default().push(sample.clone()); - Outcome::Buffered - } - } - } - } - }; - outcome = outcome.max(Some(this)); - } - match outcome { - None => self.stats.samples_query_run_only += 1, - Some(Outcome::Buffered) => self.stats.samples_accepted += 1, - Some(Outcome::FirstPane) => self.stats.samples_skipped_first_pane += 1, - Some(Outcome::Late) => self.stats.samples_dropped_late += 1, - Some(Outcome::Duplicate) => self.stats.samples_dropped_duplicate += 1, - } - self.newest_sample_ms = self.newest_sample_ms.max(Some(sample.timestamp_ms)); - if let Some(newest) = metric.and_then(|m| self.query_run_metrics.get_mut(m)) { - *newest = (*newest).max(Some(sample.timestamp_ms)); - } - } - self.close_panes(); - } - - fn close_panes(&mut self) { - let Some(watermark) = self.watermark_ms() else { - return; - }; - for builder in &mut self.builders { - let Some(built) = builder.built_through else { - continue; - }; - let pane_ms = builder.summary.definition.pane_ms; - let keep_ms = builder - .summary - .definition - .retention_ms - .saturating_add(pane_ms); - let id = &builder.summary.definition.id; - // The newest evaluation stored panes can serve is at the end of - // the newest closed pane. Panes are kept for one extra pane - // width beyond the plan's retention, so an evaluation one pane - // behind it is still served; older panes are no longer read: - // skip them. - let newest_closed_end = watermark.div_euclid(pane_ms).saturating_mul(pane_ms); - let oldest_kept_end = builder - .summary - .definition - .oldest_kept_end_ms(newest_closed_end.saturating_sub(pane_ms)); - let Some(mut end) = built.checked_add(pane_ms) else { - continue; - }; - if end < oldest_kept_end { - let Some(first_kept) = pane_end(oldest_kept_end, pane_ms) else { - continue; - }; - self.stats.panes_skipped_expired += - (first_kept.saturating_sub(end) / pane_ms) as u64; - builder.open = builder.open.split_off(&first_kept); - builder.states.drop_before(first_kept); - end = first_kept; - } - // Every pane after the first through the watermark is built, so - // a pane without samples is stored as empty, not missing. - while end <= watermark { - let scope = Scope::Ingestion { - window_start_ms: end - pane_ms, - window_end_ms: end, - revision: 1, - }; - let built = match &builder.per_sample { - // The pane's states, sealed into one batch. - Some(spec) => spec - .close(&builder.summary, &mut builder.states, end, keep_ms, &scope) - .map(|batch| SummaryInstance { - batches: vec![batch], - }), - None => { - let samples = builder.open.remove(&end).unwrap_or_default(); - builder - .summary - .run(end, &samples, scope) - .map_err(|e| e.to_string()) - } - }; - match built { - Ok(state) => { - self.store.insert( - SummaryInstanceKey::new( - id.clone(), - builder.summary.definition.selection_ending_at(end), - ), - state, - ); - self.stats.panes_built += 1; - } - Err(error) => { - tracing::error!(definition = ?id, end, %error, "pane build failed"); - self.stats.panes_failed += 1; - } - } - builder.built_through = Some(end); - match end.checked_add(pane_ms) { - Some(next) => end = next, - None => break, - } - } - self.store.evict_before( - id, - &builder - .summary - .definition - .selection_ending_at(oldest_kept_end), - ); - } - } -} - -/// Milliseconds since the Unix epoch. -fn wall_clock_ms() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)) -} - -/// End of the tumbling pane of width `pane_ms` that contains `timestamp_ms`, -/// or `None` when it does not fit in an `i64`. Panes are -/// `(end - pane_ms, end]`, like PromQL ranges: a sample at a pane boundary -/// belongs to the pane it ends. -pub fn pane_end(timestamp_ms: i64, pane_ms: i64) -> Option { - timestamp_ms - .checked_neg()? - .checked_div_euclid(pane_ms)? - .checked_neg()? - .checked_mul(pane_ms) -} - -/// Executes `dag`'s single root to completion and returns its batches. -pub fn run( - dag: &asap_executor::plan::PhysicalDAG<'_, Batch, SchemaRef>, - roots: &[u64], - scope: Scope, -) -> Result, asap_executor::Error> { - block_on(async { - let context = RunContext::new(scope, Limits::default())?; - let mut output = dag.execute(roots, context)?.remove(0); - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - let batch = batch?; - if batch.num_rows() > 0 { - batches.push((*batch).clone()); - } - } - Ok(batches) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::series_ids::intern; - - const MINUTE: i64 = 60_000; - - fn sample(job: &str, timestamp_ms: i64) -> RawSample { - RawSample { - labels: intern(&BTreeMap::from([ - ("__name__".to_string(), "latency_ms".to_string()), - ("job".to_string(), job.to_string()), - ])), - timestamp_ms, - value: 1.0, - } - } - - /// Per-sample state follows the active series: an open pane holds a - /// state only for the series that took a sample in it, and a series - /// unseen for longer than the retention plus one pane is forgotten, - /// with its cached cells. Example 4b: 10-minute KLL panes kept 60 min. - #[test] - fn per_series_state_follows_the_active_series() { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../examples/planner-layering-example4b.json"); - let workload = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); - let mut inputs = control_plane::DeploymentInputs::default(); - inputs.deployment_capabilities.raw_data_retained = false; - let plan = control_plane::plan_workload(&workload, &inputs).unwrap(); - let sds = Sds::from_plan(&plan).unwrap(); - let options = PrecomputeOptions { - allowed_lateness_ms: 0, - ..PrecomputeOptions::default() - }; - let mut engine = Precompute::new(plan.dag.dag, &sds, options).unwrap(); - assert_eq!( - ( - sds.definitions.len(), - engine.builders.len(), - engine.per_sample_definitions() - ), - (1, 1, 1) - ); - let jobs: Vec = (0..10).map(|i| format!("job{i}")).collect(); - let mut samples = Vec::new(); - for t in (MINUTE..25 * MINUTE).step_by(15_000) { - samples.extend(jobs.iter().map(|job| sample(job, t))); - } - engine.ingest_at(&samples, 300 * MINUTE); - let builder = &engine.builders[0]; - assert_eq!(builder.states.sizes(), (10, vec![10]), "(20, 30 min] open"); - let only_a: Vec = (25 * MINUTE..200 * MINUTE) - .step_by(15_000) - .map(|t| sample("job0", t)) - .collect(); - engine.ingest_at(&only_a, 300 * MINUTE); - let builder = &engine.builders[0]; - assert_eq!(builder.states.sizes(), (1, vec![1])); - assert!(builder.summary.window.cached_series() <= 1); - assert_eq!(engine.stats.panes_failed, 0); - } -} diff --git a/data_plane/src/prometheus.rs b/data_plane/src/prometheus.rs index bdee9247..d73e3d83 100644 --- a/data_plane/src/prometheus.rs +++ b/data_plane/src/prometheus.rs @@ -1,12 +1,11 @@ //! The Prometheus HTTP API as the data plane speaks it: instant-query -//! parameters and results, and fetching raw samples for query-time ranges. +//! parameters and results. use std::collections::BTreeMap; use serde_json::{json, Value}; -use crate::precompute::RawSample; -use crate::query::{BoundRawSampleWindow, ResultSample}; +use crate::query::ResultSample; /// Parameters of an instant query, from its URL query and form body. pub struct InstantQuery { @@ -68,14 +67,6 @@ pub fn bad_data(error: &str) -> Value { json!({"status": "error", "errorType": "bad_data", "error": error}) } -fn seconds_to_ms(seconds: f64) -> i64 { - (seconds * 1000.0).round() as i64 -} - -fn ms_to_seconds(ms: i64) -> String { - format!("{:.3}", ms as f64 / 1000.0) -} - /// A Prometheus instant-query response for `samples` at `time_ms`. pub fn vector_response(time_ms: i64, samples: &[ResultSample]) -> Value { let time = time_ms as f64 / 1000.0; @@ -96,96 +87,6 @@ fn format_value(value: f64) -> String { } } -/// The PromQL range selector of `range`: its metric and label matchers, every -/// value escaped as a PromQL string, over `end - start` milliseconds. -pub fn range_selector(range: &BoundRawSampleWindow) -> String { - let mut matchers = vec![format!("__name__=\"{}\"", escape(&range.metric))]; - matchers.extend( - range - .matchers - .iter() - .map(|m| format!("{}{}\"{}\"", m.name, m.op.as_str(), escape(&m.value))), - ); - format!( - "{{{}}}[{}ms]", - matchers.join(","), - range.end_ms.saturating_sub(range.start_ms) - ) -} - -/// `text` as the inside of a double-quoted PromQL string. -fn escape(text: &str) -> String { - let mut escaped = String::with_capacity(text.len()); - for c in text.chars() { - match c { - '\\' => escaped.push_str("\\\\"), - '"' => escaped.push_str("\\\""), - '\n' => escaped.push_str("\\n"), - c => escaped.push(c), - } - } - escaped -} - -/// The samples of `range`'s metric in `(start, end]`, read from Prometheus -/// with a range selector evaluated at the range's end. `headers` are sent -/// with the request: the caller's end-to-end headers, such as -/// `Authorization` and `X-Scope-OrgID`, so Prometheus authorizes and scopes -/// the read as it would the caller's own query. -pub async fn fetch( - client: &reqwest::Client, - prometheus_url: &str, - headers: &reqwest::header::HeaderMap, - range: &BoundRawSampleWindow, -) -> anyhow::Result> { - let selector = range_selector(range); - let response: Value = client - .get(format!("{prometheus_url}/api/v1/query")) - .headers(headers.clone()) - .query(&[ - ("query", selector.as_str()), - ("time", &ms_to_seconds(range.end_ms)), - ]) - .send() - .await? - .error_for_status()? - .json() - .await?; - let malformed = || anyhow::anyhow!("unexpected Prometheus response: {response}"); - let series = response["data"]["result"] - .as_array() - .ok_or_else(malformed)?; - let mut samples = Vec::new(); - for series in series { - let labels: BTreeMap = serde_json::from_value(series["metric"].clone())?; - let labels = crate::series_ids::intern(&labels); - for point in series["values"].as_array().ok_or_else(malformed)? { - let seconds = point[0].as_f64().ok_or_else(malformed)?; - let value = point[1].as_str().ok_or_else(malformed)?; - let timestamp_ms = seconds_to_ms(seconds); - // Prometheus 2 ranges include their start; the plan's do not. - if timestamp_ms <= range.start_ms { - continue; - } - samples.push(RawSample { - labels: labels.clone(), - timestamp_ms, - value: parse_value(value).ok_or_else(malformed)?, - }); - } - } - Ok(samples) -} - -fn parse_value(value: &str) -> Option { - match value { - "+Inf" | "Inf" => Some(f64::INFINITY), - "-Inf" => Some(f64::NEG_INFINITY), - "NaN" => Some(f64::NAN), - v => v.parse().ok(), - } -} - #[cfg(test)] mod tests { use super::*; @@ -213,34 +114,6 @@ mod tests { } } - /// A range selector carries the metric and the window's label matchers, - /// each value escaped as a PromQL string. - #[test] - fn builds_escaped_range_selectors() { - let range = BoundRawSampleWindow { - node: 1, - metric: "m".into(), - matchers: vec![ - crate::query::LabelMatcher { - name: "job".into(), - op: crate::query::MatchOp::Equal, - value: "a\"b\\c".into(), - }, - crate::query::LabelMatcher { - name: "env".into(), - op: crate::query::MatchOp::NotRegex, - value: "dev|test".into(), - }, - ], - start_ms: 0, - end_ms: 60_000, - }; - assert_eq!( - range_selector(&range), - r#"{__name__="m",job="a\"b\\c",env!~"dev|test"}[60000ms]"# - ); - } - /// Results use Prometheus' vector shape and value formatting. #[test] fn formats_vector_results() { @@ -261,6 +134,5 @@ mod tests { {"metric": {}, "value": [1.5, "+Inf"]}, ]}}) ); - assert_eq!(ms_to_seconds(4_200_001), "4200.001"); } } diff --git a/data_plane/src/query.rs b/data_plane/src/query.rs index 515e9893..5805e18f 100644 --- a/data_plane/src/query.rs +++ b/data_plane/src/query.rs @@ -1,88 +1,26 @@ -//! Query engine: answers a planned query at evaluation time `T` by running -//! its root of the plan's DAG over stored panes and raw samples. +//! Query engine: answers a planned query at evaluation time `T` with one +//! query run of its root of the plan's DAG, over the store. //! -//! Each root is compiled once with three kinds of inputs: -//! - stored outputs built in an ingestion run: bound to their definition's -//! summary ending at `T - offset`, which must be stored; -//! - stored outputs built in a query run (`StoredOutputBuild::QueryRun`): -//! bound to their definition's summary ending at `T - offset`. A missing -//! one is built from its coverage's raw samples, which the caller fetches, -//! and stored in the SDS for later evaluations once its pane is closed -//! (its end at or before the watermark when the evaluation chose its -//! fetch); these summary states are evicted by the plan's `retention_ms` -//! plus one pane, measured from the newest closed pane at or before the -//! watermark, as for stored outputs built in an `IngestionRun`; -//! - the plan's query-time `RawSampleWindow`s: bound to the samples in -//! `(T - offset - width, T - offset]`, which the caller fetches, selected -//! by the window's `RawSampleSelector`. -//! -//! A query the engine cannot answer returns [`Unanswered`]; the caller then -//! forwards it to Prometheus. - -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::sync::Arc; - -use asap_executor::operators::Operator; -use asap_executor::physical_planner::{compile, CompiledPhysicalDAG, InputContract, Source}; -use asap_executor::planner::ir::physical_export::{PhysicalASAPDAG, RawSampleWindow}; -use asap_executor::planner::ir::scalar::{CompareOpKind, ScalarValue}; -use asap_executor::planner::ir::schema::{DataType, FieldDataType, Schema}; -use asap_executor::planner::ir::{NonASAPOp, Operator as IrOperator, ScalarExpr}; -use asap_executor::runtime::Scope; -use asap_executor::series_labels_of_output_row; -use asap_executor::values::{Batch, SchemaRef, Value}; -use asap_planner::{InstallablePlan, StoredOutputBuild}; - -use crate::precompute::{run, RawSample, SummaryBuild, SummaryInstance}; -use crate::raw_sample_window::RawSampleWindowBatcher; -use crate::sds::{DefinitionKey, Sds, SummaryInstanceKey, SummaryStore}; - -/// A bound raw-sample window: a `RawSampleWindow` of the plan bound to one -/// evaluation time, i.e. `metric`'s raw samples in `(start_ms, end_ms]` -/// that `matchers` keep, which the caller fetches. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BoundRawSampleWindow { - pub node: u64, - pub metric: String, - /// The window's `Scan` label matchers that PromQL can express, so the - /// fetch reads only the series the plan keeps. The selector applies - /// every matcher again, so one left out only widens the fetch. - pub matchers: Vec, - pub start_ms: i64, - pub end_ms: i64, -} - -/// A PromQL label matcher: `name op "value"`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct LabelMatcher { - pub name: String, - pub op: MatchOp, - pub value: String, -} - -/// A PromQL label matching operator. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum MatchOp { - /// `=` - Equal, - /// `!=` - NotEqual, - /// `=~` - Regex, - /// `!~` - NotRegex, -} - -impl MatchOp { - pub fn as_str(self) -> &'static str { - match self { - MatchOp::Equal => "=", - MatchOp::NotEqual => "!=", - MatchOp::Regex => "=~", - MatchOp::NotRegex => "!~", - } - } -} +//! Each root is compiled once, with the materialized views it reaches +//! (without passing another view) as view slots and each `Scan` it reaches +//! outside them as a raw data slot. A run reads the views' rows tiling the +//! windows its `TimeRange`s select, and the raw samples its query-time +//! `Scan`s read. A query the engine cannot answer returns [`Unanswered`]; +//! the caller then forwards it to Prometheus (or refuses it). + +use std::collections::{BTreeMap, HashSet}; + +use asap_executor::physical_planner::{compile, CompiledPhysicalDAG}; +use asap_executor::planner::ir::schema::{Schema, PROMQL_SERIES_IDENTITY}; +use asap_executor::planner::plan::materialized_view::MaterializedView; +use asap_executor::planner::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_executor::runtime::{Limits, RunContext, Scope}; +use asap_executor::values::{Batch, Value}; +use asap_planner::InstallablePlan; +use futures::executor::block_on; +use futures::StreamExt; + +use crate::store::TimeIndexedStore; /// One series of an instant-query result. #[derive(Debug, Clone, PartialEq)] @@ -91,580 +29,181 @@ pub struct ResultSample { pub value: f64, } -/// Why the engine does not answer a query; it is then forwarded. +/// Why the engine does not answer a query. #[derive(Debug, Clone, PartialEq, thiserror::Error)] pub enum Unanswered { #[error("the installed plan has no query {0:?}")] NotPlanned(String), #[error("the data plane cannot run this query: {0}")] Unsupported(String), + /// A view's rows for a window the run reads are not stored (e.g. the + /// first window after startup, or one before its retention). #[error("{0}")] - MissingPane(String), + MissingViewRows(String), #[error("execution failed: {0}")] Execution(String), } -struct StoredInput { - node: u64, - /// The output's schema, which its panes' rows are in. - schema: SchemaRef, - definition: DefinitionKey, - offset_ms: i64, - pane_ms: i64, -} - -struct RawInput { - node: u64, - matchers: Vec, - /// The plan's `Scan` label matchers over the range's samples. - series: RawSampleWindowBatcher, - range_ms: i64, - offset_ms: i64, -} - -/// A stored output built in a `QueryRun` (`MatAtQueryKeptAcrossEvaluations`). -struct QueryRunInput { - node: u64, - schema: SchemaRef, - definition: DefinitionKey, - offset_ms: i64, - /// The output's own `TimeRange`, which its coverage's samples feed. - raw_input: u64, -} - -struct Program { - compiled: CompiledPhysicalDAG, - stored: Vec, - query_run: Vec, - /// For each `QueryRun` definition the root reads, the node its one - /// fetch is keyed by: the lowest `TimeRange` among its outputs. - query_run_fetch_node: BTreeMap, - raw: Vec, - output: Schema, -} - -/// What happened to the stored outputs built in a `QueryRun` evaluations read. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize)] -pub struct QueryRunStats { - /// Summary states built from raw samples because the SDS had none. - pub built: u64, - /// Of those, the ones stored in the SDS: their coverage was closed. - pub stored: u64, - /// Of those stored, the ones whose own metric's newest sample had not - /// passed the coverage's end: the watermark is global, so a metric - /// lagging behind others can have a summary state stored before all its samples - /// arrived. A per-metric watermark is the follow-up. - pub stored_while_metric_behind: u64, - /// Summaries read from an earlier evaluation instead of rebuilt. - pub reused: u64, -} - -/// What ingestion had seen when an evaluation decided what to fetch. The -/// store decision uses this snapshot, not the state when the evaluation -/// runs: samples ingested in between were not in the fetched data. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct IngestionProgress { - /// The precompute engine's watermark; `None` before any sample. - pub watermark_ms: Option, - /// The newest sample time of each metric stored outputs built in a `QueryRun` read. - pub newest_sample_ms: BTreeMap, -} - -/// One raw read an evaluation needs, prepared by [`QueryEngine::bound_raw_sample_windows`]. -#[derive(Debug, Clone)] -pub struct Prepared { - /// The root answering the query. - pub root: usize, - /// The bound raw-sample windows to fetch. - pub windows: Vec, - /// What ingestion had seen when `ranges` were chosen. - pub progress: IngestionProgress, -} - pub struct QueryEngine { - /// How many summary states of stored outputs built in a `QueryRun` - /// evaluations built, stored and reused. - pub query_run_stats: QueryRunStats, - /// How each `QueryRun` definition's summaries are built, or why they - /// cannot be. - query_run_builds: BTreeMap>, - /// The label matchers of each `QueryRun` definition's `Scan`, pushed into - /// its fetch. - query_run_matchers: BTreeMap>, /// Each query, with its compiled root or why it cannot run. - roots: Vec<(String, Result)>, + roots: Vec<(String, Result)>, + limits: Limits, } impl QueryEngine { - pub fn new(plan: &InstallablePlan, sds: &Sds) -> Self { - let dag = &plan.dag.dag; - let windows: HashMap = plan - .query_time_raw_windows - .iter() - .map(|w| (w.time_range_node_id as u64, w)) - .collect(); - let query_run_builds: BTreeMap<_, _> = sds - .definitions - .values() - .filter(|d| d.built_at == StoredOutputBuild::QueryRun) - .map(|d| { - let build = SummaryBuild::new(dag, d).map_err(|e| e.to_string()); - (d.key(), build) - }) - .collect(); + /// Compiles each query's root of `plan`, whose views are `views`. + pub fn new(plan: &InstallablePlan, views: &[MaterializedView], limits: Limits) -> Self { let roots = plan - .queries + .query_texts .iter() - .map(|q| { - let program = program(dag, sds, &query_run_builds, &windows, q.root_node_id as u64); - (q.query.trim().to_string(), program) - }) - .collect(); - let query_run_matchers = query_run_builds - .keys() - .map(|id| { - let matchers = sds.definitions[id] - .window - .as_ref() - .map(|w| label_matchers(dag, w.scan_node_id as u64)) - .unwrap_or_default(); - (id.clone(), matchers) + .zip(&plan.dag.roots) + .map(|(query, &root)| { + let read = views_read(&plan.dag, views, root); + let compiled = compile(&plan.dag, &read, &[root as u64]).map_err(|e| e.to_string()); + (query.trim().to_string(), compiled) }) .collect(); - Self { - query_run_stats: QueryRunStats::default(), - query_run_builds, - query_run_matchers, - roots, - } - } - - /// The root answering `query` at `time_ms`, and the raw samples it needs: - /// its query-time windows, and for each `QueryRun` definition one range - /// covering every summary it reads that is not stored. Fails if the - /// query is not planned, a time does not fit, or an ingestion-built - /// summary it reads is missing. `progress` is what ingestion has seen - /// now; pass the returned copy to [`Self::evaluate`]. - pub fn bound_raw_sample_windows( - &self, - query: &str, - time_ms: i64, - store: &SummaryStore, - progress: IngestionProgress, - ) -> Result { - let (root, program) = self.program(query)?; - for input in &program.stored { - pane(input, time_ms, store)?; - } - let mut ranges: Vec = program - .raw - .iter() - .map(|input| { - let (start_ms, end_ms) = window(time_ms, input.offset_ms, input.range_ms)?; - Ok(BoundRawSampleWindow { - node: input.node, - metric: input.series.metric().to_string(), - matchers: input.matchers.clone(), - start_ms, - end_ms, - }) - }) - .collect::>()?; - // One fetch per `QueryRun` definition, covering all its missing - // summaries. - let mut missing: BTreeMap<&DefinitionKey, (i64, i64)> = BTreeMap::new(); - for input in &program.query_run { - let build = self.query_run_build(&input.definition); - let (start_ms, end_ms) = window(time_ms, input.offset_ms, build.definition.pane_ms)?; - if store - .get( - &input.definition.id, - &input.definition.selection_ending_at(end_ms), - ) - .is_none() - { - let span = missing - .entry(&input.definition) - .or_insert((start_ms, end_ms)); - *span = (span.0.min(start_ms), span.1.max(end_ms)); - } - } - for (definition, (start_ms, end_ms)) in missing { - let build = self.query_run_build(definition); - ranges.push(BoundRawSampleWindow { - node: program.query_run_fetch_node[definition], - metric: build.metric().to_string(), - matchers: self.query_run_matchers[definition].clone(), - start_ms, - end_ms, - }); - } - Ok(Prepared { - root, - windows: ranges, - progress, - }) - } - - /// Runs `root` at `time_ms` over stored summaries and `raw`, the samples - /// of each of its [`BoundRawSampleWindow`]s by node. A stored output built in a `QueryRun` not stored - /// is built from its raw samples and stored in `store`, under the same - /// key as those built in an `IngestionRun`, if its coverage was closed in - /// `progress` (the snapshot [`Self::bound_raw_sample_windows`] chose the fetch by). - /// Then stored outputs built in a `QueryRun` older than their definition's retention, - /// measured from the newest closed pane at or before the watermark, are - /// evicted. - pub fn evaluate( - &mut self, - root: usize, - time_ms: i64, - store: &mut SummaryStore, - progress: &IngestionProgress, - raw: &BTreeMap>, - ) -> Result, Unanswered> { - let program = self.roots[root] - .1 - .as_ref() - .map_err(|reason| Unanswered::Unsupported(reason.clone()))?; - let failed = |e: asap_executor::Error| Unanswered::Execution(e.to_string()); - let scope = Scope::Query { - evaluation_time_ms: time_ms, - revision: 1, - }; - let mut sources: BTreeMap> = BTreeMap::new(); - for input in &program.stored { - let pane = pane(input, time_ms, store)?; - let source = - Operator::source(input.schema.clone(), pane.batches.clone()).map_err(failed)?; - sources.insert(input.node, Box::new(source)); - } - for input in &program.query_run { - let build = self.query_run_builds[&input.definition] - .as_ref() - .expect("compiled roots read only `QueryRun` definitions that build"); - let (_, end_ms) = window(time_ms, input.offset_ms, build.definition.pane_ms)?; - let selection = input.definition.selection_ending_at(end_ms); - let summary = match store.get(&input.definition.id, &selection) { - Some(summary) => { - self.query_run_stats.reused += 1; - summary.clone() - } - None => { - // `bound_raw_sample_windows` asked for these samples unless the - // summary was stored then and evicted since. - let samples = raw - .get(&program.query_run_fetch_node[&input.definition]) - .ok_or_else(|| { - Unanswered::MissingPane(format!( - "stored output built in a `QueryRun` ending at {end_ms} of {:?} was evicted", - input.definition - )) - })?; - // Built as at its coverage's end, so every evaluation - // that reads it reads the same summary. - let build_scope = Scope::Query { - evaluation_time_ms: end_ms, - revision: 1, - }; - let summary = build - .run(end_ms, samples, build_scope) - .map_err(|e| Unanswered::Execution(e.to_string()))?; - self.query_run_stats.built += 1; - // Stored only once closed, as the precompute engine closes - // panes: a sample may still arrive for an open one. - if progress.watermark_ms.is_some_and(|w| end_ms <= w) { - self.query_run_stats.stored += 1; - let metric_newest = progress.newest_sample_ms.get(build.metric()); - if metric_newest.is_none_or(|newest| *newest < end_ms) { - self.query_run_stats.stored_while_metric_behind += 1; - } - store.insert( - SummaryInstanceKey::new(input.definition.id.clone(), selection), - summary.clone(), - ); - } - summary - } - }; - let source = Operator::source(input.schema.clone(), summary.batches).map_err(failed)?; - sources.insert(input.node, Box::new(source)); - } - if let Some(watermark) = progress.watermark_ms { - for build in self.query_run_builds.values().flatten() { - let definition = &build.definition; - let pane_ms = definition.pane_ms; - // As for stored outputs built in an `IngestionRun`: measured from the newest - // closed coverage, one pane of margin. - let newest_closed_end = watermark.div_euclid(pane_ms).saturating_mul(pane_ms); - let oldest = - definition.oldest_kept_end_ms(newest_closed_end.saturating_sub(pane_ms)); - store.evict_before(&definition.id, &definition.selection_ending_at(oldest)); - } - } - for input in &program.raw { - let samples = raw.get(&input.node).map(Vec::as_slice).unwrap_or_default(); - let (_, end_ms) = window(time_ms, input.offset_ms, input.range_ms)?; - let rows = input.series.batch(end_ms, samples).map_err(failed)?; - let source = - Operator::source(input.series.schema.clone(), vec![rows]).map_err(failed)?; - sources.insert(input.node, Box::new(source)); - } - let dag = program.compiled.instantiate(sources).map_err(failed)?; - let batches = run(&dag, program.compiled.roots(), scope).map_err(failed)?; - result_samples(&program.output, &batches) + Self { roots, limits } } /// The planned queries the engine cannot run, with why. pub fn unexecutable(&self) -> Vec<(String, String)> { self.roots .iter() - .filter_map(|(query, program)| Some((query.clone(), program.as_ref().err()?.clone()))) + .filter_map(|(query, root)| Some((query.clone(), root.as_ref().err()?.clone()))) .collect() } - /// The build of a `QueryRun` definition a compiled root reads. - fn query_run_build(&self, definition: &DefinitionKey) -> &SummaryBuild { - self.query_run_builds[definition] - .as_ref() - .expect("compiled roots read only `QueryRun` definitions that build") - } - - fn program(&self, query: &str) -> Result<(usize, &Program), Unanswered> { + /// Answers `query` at `time_ms` from `store`. + pub fn evaluate( + &self, + query: &str, + time_ms: i64, + store: &TimeIndexedStore, + ) -> Result, Unanswered> { let query = query.trim(); - let root = self + let compiled = self .roots .iter() - .position(|(planned, _)| planned == query) - .ok_or_else(|| Unanswered::NotPlanned(query.to_string()))?; - match &self.roots[root].1 { - Ok(program) => Ok((root, program)), - Err(reason) => Err(Unanswered::Unsupported(reason.clone())), - } + .find(|(planned, _)| planned == query) + .ok_or_else(|| Unanswered::NotPlanned(query.to_string()))? + .1 + .as_ref() + .map_err(|reason| Unanswered::Unsupported(reason.clone()))?; + let run_store = store.run_store(); + let batches = run(compiled, run_store.clone(), time_ms, &self.limits); + let batches = batches.map_err(|error| match run_store.missing_view_rows() { + Some(missing) => Unanswered::MissingViewRows(missing), + None => Unanswered::Execution(error.to_string()), + })?; + let root = compiled.roots()[0]; + let schema = compiled + .output_schema(root) + .map_err(|e| Unanswered::Execution(e.to_string()))?; + result_samples(&schema, &batches) } } -/// `(start, end]` of a window `width_ms` wide ending `offset_ms` before -/// `time_ms`, or why it does not fit in `i64` milliseconds. -fn window(time_ms: i64, offset_ms: i64, width_ms: i64) -> Result<(i64, i64), Unanswered> { - let out_of_range = - || Unanswered::Unsupported(format!("evaluation time {time_ms} ms is out of range")); - let end = time_ms.checked_sub(offset_ms).ok_or_else(out_of_range)?; - let start = end.checked_sub(width_ms).ok_or_else(out_of_range)?; - Ok((start, end)) -} - -/// The stored pane `input` reads at `time_ms`. -fn pane<'a>( - input: &StoredInput, +/// The batches of `compiled`'s one root in a query run at `time_ms`. +fn run( + compiled: &CompiledPhysicalDAG, + store: std::sync::Arc, time_ms: i64, - store: &'a SummaryStore, -) -> Result<&'a SummaryInstance, Unanswered> { - let (_, end) = window(time_ms, input.offset_ms, input.pane_ms)?; - if end.rem_euclid(input.pane_ms) != 0 { - return Err(Unanswered::MissingPane(format!( - "{time_ms} is not on the {} ms pane grid", - input.pane_ms - ))); - } - store - .get( - &input.definition.id, - &input.definition.selection_ending_at(end), - ) - .ok_or_else(|| { - Unanswered::MissingPane(format!( - "pane ending at {end} of {:?} is not stored", - input.definition - )) - }) + limits: &Limits, +) -> Result, asap_executor::Error> { + let instance = compiled.instantiate(store)?; + let context = RunContext::new( + Scope::Query { + evaluation_time_ms: time_ms, + }, + limits.clone(), + )?; + block_on(async { + let mut output = instance.execute(compiled.roots(), context)?.remove(0); + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + let batch = batch?; + if batch.num_rows() > 0 { + batches.push((*batch).clone()); + } + } + Ok(batches) + }) } -/// Compiles `root` with its stored and raw inputs, or says why it cannot. -fn program( +/// The views `root` reaches without passing another view (itself +/// included, when it is one): the view slots of its query run. +fn views_read( dag: &PhysicalASAPDAG, - sds: &Sds, - query_run_builds: &BTreeMap>, - windows: &HashMap, - root: u64, -) -> Result { - let nodes: HashMap = dag.nodes.iter().map(|n| (n.id as u64, n)).collect(); - let mut producers: HashMap> = HashMap::new(); - for edge in &dag.edges { - producers - .entry(edge.consumer as u64) - .or_default() - .push(edge.producer as u64); - } - // A query-time window caches a series for as long as any summary is - // kept. - let longest_retention_ms = sds - .definitions - .values() - .map(|d| d.retention_ms) - .max() - .unwrap_or(0); - let (mut stored, mut query_run, mut raw) = (Vec::new(), Vec::new(), Vec::new()); + views: &[MaterializedView], + root: PhysicalASAPNodeId, +) -> Vec { + let view_ids: HashSet = views.iter().map(|v| v.node_id).collect(); + let mut read = HashSet::new(); + let mut seen = HashSet::new(); let mut pending = vec![root]; - let mut seen = BTreeSet::new(); while let Some(id) = pending.pop() { if !seen.insert(id) { continue; } - if let Some((definition, binding)) = sds.outputs.get(&id) { - match sds.definitions[definition].built_at { - StoredOutputBuild::IngestionRun => stored.push(StoredInput { - node: id, - schema: Arc::new(nodes[&id].output_schema.clone()), - definition: definition.clone(), - offset_ms: binding.offset_ms, - pane_ms: sds.definitions[definition].pane_ms, - }), - StoredOutputBuild::QueryRun => { - query_run_builds[definition] - .as_ref() - .map_err(Clone::clone)?; - query_run.push(QueryRunInput { - node: id, - schema: Arc::new(nodes[&id].output_schema.clone()), - definition: definition.clone(), - offset_ms: binding.offset_ms, - raw_input: binding.raw_input, - }); - } - } + if view_ids.contains(&id) { + read.insert(id); continue; } - match &nodes[&id].payload { - IrOperator::NonASAP(NonASAPOp::TimeRange { .. }) => { - let window = windows.get(&id).ok_or_else(|| { - format!("node {id}: a TimeRange the plan lists no window for") - })?; - let series = RawSampleWindowBatcher::new(dag, window, longest_retention_ms) - .map_err(|e| format!("node {id}: {e}"))?; - raw.push(RawInput { - node: id, - matchers: label_matchers(dag, window.scan_node_id as u64), - series, - range_ms: window.width_ms, - offset_ms: window.end_offset_ms, - }); - continue; - } - IrOperator::NonASAP(NonASAPOp::Scan { .. }) => { - return Err(format!("node {id}: a Scan read without a TimeRange")) - } - _ => pending.extend(producers.get(&id).into_iter().flatten().copied()), - } - } - let mut inputs = BTreeMap::new(); - for node in stored - .iter() - .map(|i| i.node) - .chain(query_run.iter().map(|i| i.node)) - { - let schema = Arc::new(nodes[&node].output_schema.clone()); - inputs.insert(node, InputContract::bounded(schema)); - } - for input in &raw { - inputs.insert( - input.node, - InputContract::bounded(input.series.schema.clone()), + pending.extend( + dag.edges + .iter() + .filter(|e| e.consumer == id) + .map(|e| e.producer), ); } - let compiled = compile(dag, inputs, &[root]).map_err(|e| e.to_string())?; - let mut query_run_fetch_node: BTreeMap = BTreeMap::new(); - for input in &query_run { - let node = query_run_fetch_node - .entry(input.definition.clone()) - .or_insert(input.raw_input); - *node = (*node).min(input.raw_input); - } - Ok(Program { - compiled, - stored, - query_run, - query_run_fetch_node, - raw, - output: nodes[&root].output_schema.clone(), - }) -} - -/// The label matchers of the `Scan` at `scan` that PromQL can express: a -/// label column compared (`=`, `!=`, `=~`, `!~`) with a text literal. -/// Other predicates are left out of the fetch; the selector applies them. -fn label_matchers(dag: &PhysicalASAPDAG, scan: u64) -> Vec { - let Some(node) = dag.nodes.iter().find(|n| n.id as u64 == scan) else { - return Vec::new(); - }; - let IrOperator::NonASAP(NonASAPOp::Scan { - predicates, schema, .. - }) = &node.payload - else { - return Vec::new(); - }; - predicates + views .iter() - .filter_map(|predicate| { - let ScalarExpr::Compare { - left, op, right, .. - } = &predicate.0 - else { - return None; - }; - let (ScalarExpr::Column(column), ScalarExpr::Literal(ScalarValue::Utf8(value))) = - (left.as_ref(), right.as_ref()) - else { - return None; - }; - let op = match op { - CompareOpKind::Eq => MatchOp::Equal, - CompareOpKind::Ne => MatchOp::NotEqual, - CompareOpKind::Regex => MatchOp::Regex, - CompareOpKind::NotRegex => MatchOp::NotRegex, - _ => return None, - }; - let name = &schema.fields.get(*column)?.name; - // Only Prometheus label names; the series identity column and - // other derived columns are not labels. - let is_label = !name.is_empty() - && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - && !name.starts_with(|c: char| c.is_ascii_digit()); - is_label.then(|| LabelMatcher { - name: name.clone(), - op, - value: value.clone(), - }) - }) + .filter(|v| read.contains(&v.node_id)) + .cloned() .collect() } /// PromQL vector samples from a root's result batches: labels from the -/// series identity and the other text columns, the value from the one -/// `Float64` column. The one place rows are read: the answer's JSON -/// encoding. +/// series identity and every other non-empty text column, the value from +/// the one `Float64` column. fn result_samples(schema: &Schema, batches: &[Batch]) -> Result, Unanswered> { - let unsupported = |reason: &str| Unanswered::Unsupported(format!("result rows: {reason}")); - let values: Vec = schema + let floats = schema .fields .iter() - .enumerate() - .filter(|(_, f)| f.dtype == FieldDataType::Plain(DataType::Float64)) - .map(|(i, _)| i) - .collect(); - let [value] = values[..] else { - return Err(unsupported("expected one Float64 value column")); - }; - batches - .iter() - .flat_map(Batch::to_rows) - .map(|row| { - let labels = series_labels_of_output_row(schema, &row) - .map_err(|e| Unanswered::Execution(e.to_string()))?; - let value = match &row[value] { - Value::Float64(v) => *v, - other => return Err(Unanswered::Execution(format!("value {other:?}"))), - }; - Ok(ResultSample { labels, value }) + .filter(|f| { + f.dtype + == asap_executor::planner::ir::schema::FieldDataType::Plain( + asap_executor::planner::ir::schema::DataType::Float64, + ) }) - .collect() + .count(); + if floats != 1 { + return Err(Unanswered::Unsupported( + "result rows: expected one Float64 value column".into(), + )); + } + let mut samples = Vec::new(); + for row in batches.iter().flat_map(Batch::to_rows) { + let mut labels = BTreeMap::new(); + let mut value = None; + for (field, cell) in schema.fields.iter().zip(&row) { + match cell { + Value::Utf8(text) if field.name == PROMQL_SERIES_IDENTITY => { + let identity: BTreeMap = serde_json::from_str(text) + .map_err(|e| Unanswered::Execution(format!("series identity: {e}")))?; + labels.extend(identity); + } + Value::Utf8(text) if !text.is_empty() => { + labels.insert(field.name.clone(), text.to_string()); + } + Value::Float64(v) => value = Some(*v), + _ => {} + } + } + let value = + value.ok_or_else(|| Unanswered::Execution("a result row without a value".into()))?; + samples.push(ResultSample { labels, value }); + } + Ok(samples) } diff --git a/data_plane/src/raw_sample_window.rs b/data_plane/src/raw_sample_window.rs deleted file mode 100644 index 31a853b8..00000000 --- a/data_plane/src/raw_sample_window.rs +++ /dev/null @@ -1,318 +0,0 @@ -//! Raw samples to a `RawSampleWindow`'s input batch, column by column. -//! -//! A raw-sample window of the plan (`Scan -> TimeShift* -> TimeRange`) -//! reads samples of one metric. Each series is looked at once: the -//! executor's `RawSampleSelector` applies the `Scan`'s label matchers to -//! it, and its text cells (its series identity and label columns) are -//! computed and cached. A window's batch is then built from the samples' -//! times and values and their series' cached cells, as Arrow columns -//! (`Batch::try_from_columns`): the columns the selector builds, without -//! a `Sample` or a row per sample. - -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::{Arc, RwLock}; - -use asap_executor::arrow::array::{ - ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array, -}; -use asap_executor::planner::ir::physical_export::{PhysicalASAPDAG, RawSampleWindow}; -use asap_executor::planner::ir::schema::{DataType, FieldDataType}; -use asap_executor::planner::ir::{NonASAPOp, Operator, TimeRangeKind}; -use asap_executor::values::{Batch, BatchColumn, SchemaRef}; -use asap_executor::{series_identity, RawSampleSelector, Sample}; - -use crate::precompute::RawSample; - -/// The `Scan` schema's column that holds a series' full identity. -pub(crate) const SERIES_IDENTITY_COLUMN: &str = "$promql_series_identity"; - -/// The Prometheus stale marker: a NaN that ends a series, not a sample. -pub(crate) const STALE_MARKER: u64 = 0x7ff0_0000_0000_0002; - -/// Whether `value` is the Prometheus stale marker. -pub(crate) fn is_stale(value: f64) -> bool { - value.to_bits() == STALE_MARKER -} - -/// What one column of a window's input holds. -#[derive(Debug, Clone, PartialEq)] -enum ColumnKind { - Time, - Value, - /// The series identity, or a label (empty when the series lacks it). - Text, -} - -/// One selected series' text cells, by input column (`None` for the time -/// and value columns). -#[derive(Debug, PartialEq)] -pub(crate) struct SeriesCells { - pub(crate) texts: Vec>>, -} - -/// Each series' cells, or `None` when the `Scan` does not select it, with -/// the newest sample time the series was looked up with, by series id. -type SeriesCache = ahash::HashMap>, AtomicI64)>; - -/// One `RawSampleWindow` of the plan, prepared to turn raw samples into its -/// input batch. A series is forgotten once it has not been seen for -/// `keep_ms`, so the cache holds the series of recent windows, not every -/// series ever seen. -pub(crate) struct RawSampleWindowBatcher { - selector: RawSampleSelector, - /// The `TimeRange`'s output schema, which the batch is in. - pub(crate) schema: SchemaRef, - kinds: Vec, - width_ms: i64, - /// An instant selector keeps only each series' latest sample. - latest_sample_per_series: bool, - series: RwLock, - /// How long a series not seen stays cached, past the window's width. - keep_ms: i64, - /// The window end series were last forgotten at. - forgotten_at_ms: AtomicI64, -} - -impl RawSampleWindowBatcher { - /// The window `window` of `dag`, which caches a series until it has - /// not been seen for `keep_ms` past the window's width. - pub(crate) fn new( - dag: &PhysicalASAPDAG, - window: &RawSampleWindow, - keep_ms: i64, - ) -> Result { - let invalid = |message: &str| asap_executor::Error::Invalid(message.into()); - let selector = RawSampleSelector::new(dag, window)?; - let time_range = dag - .nodes - .iter() - .find(|n| n.id == window.time_range_node_id) - .ok_or_else(|| invalid("window names an unknown TimeRange"))?; - let Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) = &time_range.payload else { - return Err(invalid("window's time range is not a TimeRange")); - }; - let schema = Arc::new(time_range.output_schema.clone()); - let kinds = schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - if Some(index) == schema.time_index { - Ok(ColumnKind::Time) - } else if field.name == "value" - && field.dtype == FieldDataType::Plain(DataType::Float64) - { - Ok(ColumnKind::Value) - } else if field.dtype == FieldDataType::Plain(DataType::Utf8) { - Ok(ColumnKind::Text) - } else { - Err(invalid("unsupported PromQL source column")) - } - }) - .collect::, _>>()?; - Ok(Self { - selector, - schema, - kinds, - width_ms: window.width_ms, - latest_sample_per_series: *kind == TimeRangeKind::Instant, - series: RwLock::new(ahash::HashMap::default()), - keep_ms, - forgotten_at_ms: AtomicI64::new(i64::MIN), - }) - } - - /// The metric the window reads. - pub(crate) fn metric(&self) -> &str { - self.selector.metric() - } - - /// The cells of `sample`'s series, or `None` when the `Scan` does not - /// select it; computed once per series. - pub(crate) fn cells(&self, sample: &RawSample) -> Option> { - if let Some((cells, newest_ms)) = self - .series - .read() - .unwrap_or_else(|e| e.into_inner()) - .get(&sample.labels.id) - { - newest_ms.fetch_max(sample.timestamp_ms, Ordering::Relaxed); - return cells.clone(); - } - let cells = self.series_cells(sample); - self.series - .write() - .unwrap_or_else(|e| e.into_inner()) - .insert( - sample.labels.id, - (cells.clone(), AtomicI64::new(sample.timestamp_ms)), - ); - cells - } - - /// How many series are cached. - #[cfg(test)] - pub(crate) fn cached_series(&self) -> usize { - self.series.read().unwrap_or_else(|e| e.into_inner()).len() - } - - /// Forgets the series whose newest sample is at or before `oldest_ms`. - pub(crate) fn forget_unseen(&self, oldest_ms: i64) { - self.forgotten_at_ms.fetch_max(oldest_ms, Ordering::Relaxed); - self.series - .write() - .unwrap_or_else(|e| e.into_inner()) - .retain(|_, (_, newest_ms)| newest_ms.load(Ordering::Relaxed) > oldest_ms); - } - - /// Forgets the series not seen for `keep_ms` past the width of the - /// window ending at `end_ms`, at most once per `keep_ms` of window ends, - /// so a scan of the cache is amortized over many windows. - fn forget_old(&self, end_ms: i64) { - let oldest_ms = end_ms - .saturating_sub(self.width_ms) - .saturating_sub(self.keep_ms); - let last = self.forgotten_at_ms.load(Ordering::Relaxed); - if oldest_ms > last.saturating_add(self.keep_ms.max(1)) { - self.forget_unseen(oldest_ms); - } - } - - fn series_cells(&self, sample: &RawSample) -> Option> { - // The selector applies the Scan's matchers to the series' labels; - // the value is not the sample's, which may be a stale marker. - let one = Sample { - labels: (**sample.labels).clone(), - timestamp_ms: sample.timestamp_ms, - value: 0.0, - }; - let selected = self - .selector - .window_batch(sample.timestamp_ms, &[one]) - .ok()?; - if selected.num_rows() != 1 { - return None; - } - let identity: Arc = series_identity(&sample.labels).ok()?.into(); - let texts = self - .kinds - .iter() - .zip(&self.schema.fields) - .map(|(kind, field)| match kind { - ColumnKind::Text if field.name == SERIES_IDENTITY_COLUMN => Some(identity.clone()), - ColumnKind::Text => Some( - sample - .labels - .get(&field.name) - .map_or_else(|| Arc::from(""), |v| Arc::from(v.as_str())), - ), - _ => None, - }) - .collect(); - Some(Arc::new(SeriesCells { texts })) - } - - /// The window's input batch for the window ending at `end_ms`, as the - /// executor's `RawSampleSelector::window_batch` gives it: the samples - /// in `(end_ms - width, end_ms]` of the series the `Scan` selects - /// (only each series' latest for an instant selector), in the - /// `TimeRange`'s schema. A stale marker is never a row: a range window - /// drops it, and an instant selector whose latest sample is one has no - /// row for that series. - pub(crate) fn batch( - &self, - end_ms: i64, - samples: &[RawSample], - ) -> Result { - let start_ms = end_ms - self.width_ms; - let in_window = samples - .iter() - .filter(|s| start_ms < s.timestamp_ms && s.timestamp_ms <= end_ms); - let mut points: Vec<(Arc, i64, f64)> = Vec::new(); - if self.latest_sample_per_series { - let mut latest: ahash::HashMap = ahash::HashMap::default(); - for sample in in_window { - let entry = latest.entry(sample.labels.id).or_insert(sample); - if sample.timestamp_ms >= entry.timestamp_ms { - *entry = sample; - } - } - // In label order, as the selector emits them. - let mut latest: Vec<&RawSample> = latest.into_values().collect(); - latest.sort_unstable_by(|a, b| a.labels.cmp(&b.labels)); - for sample in latest.into_iter().filter(|s| !is_stale(s.value)) { - if let Some(cells) = self.cells(sample) { - points.push((cells, sample.timestamp_ms, sample.value)); - } - } - } else { - // Samples of one series are together: look each up once. - let mut current: Option<(u32, Option>)> = None; - for sample in in_window.filter(|s| !is_stale(s.value)) { - let cells = match ¤t { - Some((id, cells)) if *id == sample.labels.id => cells.clone(), - _ => { - let cells = self.cells(sample); - current = Some((sample.labels.id, cells.clone())); - cells - } - }; - if let Some(cells) = cells { - points.push((cells, sample.timestamp_ms, sample.value)); - } - } - } - self.forget_old(end_ms); - self.batch_of(&points) - } - - /// The batch of `points`: each sample's series cells, time and value. - pub(crate) fn batch_of( - &self, - points: &[(Arc, i64, f64)], - ) -> Result { - let columns = self - .kinds - .iter() - .enumerate() - .map(|(index, kind)| { - let array: ArrayRef = match kind { - ColumnKind::Time => Arc::new(TimestampMillisecondArray::from_iter_values( - points.iter().map(|(_, time, _)| *time), - )), - ColumnKind::Value => Arc::new(Float64Array::from_iter_values( - points.iter().map(|(_, _, value)| *value), - )), - ColumnKind::Text => text_column( - points - .iter() - .map(|(cells, _, _)| cells.texts[index].as_deref().unwrap_or("")), - )?, - }; - Ok(BatchColumn::Plain(array)) - }) - .collect::, asap_executor::Error>>()?; - Batch::try_from_columns(self.schema.clone(), columns) - } -} - -/// A text column in the batch layout: a dictionary of the distinct texts, -/// with each row's code. -pub(crate) fn text_column<'a>( - texts: impl Iterator, -) -> Result { - let mut codes: ahash::HashMap<&str, u32> = ahash::HashMap::default(); - let mut values: Vec<&str> = Vec::new(); - let keys: UInt32Array = texts - .map(|text| { - let next = values.len() as u32; - *codes.entry(text).or_insert_with(|| { - values.push(text); - next - }) - }) - .collect(); - let dictionary = DictionaryArray::try_new(keys, Arc::new(StringArray::from(values))) - .map_err(|e| asap_executor::Error::Invalid(e.to_string()))?; - Ok(Arc::new(dictionary)) -} diff --git a/data_plane/src/sds.rs b/data_plane/src/sds.rs deleted file mode 100644 index 364c3ef2..00000000 --- a/data_plane/src/sds.rs +++ /dev/null @@ -1,452 +0,0 @@ -//! Stored summaries (SDS): the summary instances the data plane stores for a -//! plan, and how a query finds them again. -//! -//! The installed plan lists every output it stores across evaluations -//! (`InstallablePlan::stored_outputs`): which run builds it (an -//! `IngestionRun`, or the `QueryRun` of the first evaluation that reads it), -//! its `SummaryCoverage` (the plan's definition id and selection), the -//! `RawSampleWindow`s and stored outputs its build reads, and its -//! `retention_ms`. -//! -//! A definition's identity is the plan's definition id. Outputs with the same -//! definition id, the same label constraints in their selection and the -//! same pane width differ only in where their time interval ends, so they -//! share one [`SummaryDefinition`]: the data plane builds each pane once and -//! every output reads the one it needs. Outputs with the same definition id -//! but other label constraints or another pane width (e.g. one query's -//! 1-second panes and another's 30-second panes of the same per-series sum) -//! are other [`SummaryDefinition`]s ([`DefinitionKey`]). A summary instance -//! is keyed by its definition id and its concrete [`Selection`]: those label -//! constraints and the time interval `(end - width, end]` the plan's -//! relative time binds to at evaluation time. -//! -//! A stored output may be built from other stored outputs (rollups over time -//! or across series). [`Sds::ingestion_build_order`] orders definitions so -//! that each one's stored inputs come first; the precompute engine refuses -//! such a definition until it can build it once all its stored inputs are -//! sealed. - -use std::collections::{BTreeMap, HashMap}; - -use std::ops::Bound; -use std::sync::Arc; - -use asap_executor::planner::ir::physical_export::{PhysicalASAPDAGNode, RawSampleWindow}; -use asap_executor::planner::ir::properties::summary_coverage::SelectionBox; -use asap_executor::planner::ir::schema::Schema; -use asap_planner::{InstallablePlan, StoredOutputBuild}; - -/// What a stored summary computes: the plan's -/// `StoredCoverage::definition_id`, equal on every node, plan and process -/// that computes the same thing, whatever its selection. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct DefinitionId(pub String); - -/// The label constraints of a selection (its `SelectionBox` columns), in the -/// plan's JSON form; empty when it has none. -pub type LabelConstraints = Arc; - -/// One [`SummaryDefinition`] of the SDS: a definition id, and the label -/// constraints and pane width its outputs share. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct DefinitionKey { - pub id: DefinitionId, - pub labels: LabelConstraints, - pub pane_ms: i64, -} - -impl DefinitionKey { - /// `id`'s panes `pane_ms` wide with no label constraints. - pub fn unconstrained(id: DefinitionId, pane_ms: i64) -> Self { - Self { - id, - labels: Arc::from(""), - pane_ms, - } - } -} - -/// One computation the data plane stores pane by pane, for one set of label -/// constraints. -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryDefinition { - pub id: DefinitionId, - /// The label constraints of its outputs' selection. - pub labels: LabelConstraints, - /// Which run builds its stored outputs (`StoredOutputBuild`): an - /// `IngestionRun`, or the `QueryRun` of the first evaluation that needs - /// one (stored once its coverage is closed). `IngestionRun` when any - /// output of the definition says so. - pub built_at: StoredOutputBuild, - /// Pane width: each summary instance covers `(end - pane_ms, end]`. - pub pane_ms: i64, - /// The output that builds a pane; any output of the definition computes - /// the same, and this is the first in node order. - pub builder: u64, - /// The `RawSampleWindow` the builder reads: its `TimeRange`, fed one - /// pane's raw samples, and the `Scan` that selects them. `None` for a - /// definition built from other stored outputs only. - pub window: Option, - pub output_schema: Schema, - /// The definitions of the stored outputs the builder reads, which must - /// be built first. Empty while every stored output reads raw samples. - pub stored_inputs: Vec, - /// How far back from the newest evaluation time a summary stays available: - /// the largest `retention_ms` the plan states for the definition's - /// outputs. A pane that starts earlier is no longer read. - pub retention_ms: i64, -} - -impl SummaryDefinition { - /// Its key in [`Sds::definitions`]. - pub fn key(&self) -> DefinitionKey { - DefinitionKey { - id: self.id.clone(), - labels: self.labels.clone(), - pane_ms: self.pane_ms, - } - } - - /// The concrete selection of its pane ending at `end_ms`. - pub fn selection_ending_at(&self, end_ms: i64) -> Selection { - self.key().selection_ending_at(end_ms) - } - - /// The builder's `TimeRange`, which one pane's raw samples feed. - pub fn raw_input(&self) -> Option { - self.window.as_ref().map(|w| w.time_range_node_id as u64) - } - - /// The end of the oldest pane still read when the newest evaluation is - /// at `newest_ms`: panes ending before it are evicted. - pub fn oldest_kept_end_ms(&self, newest_ms: i64) -> i64 { - newest_ms - .saturating_sub(self.retention_ms) - .saturating_add(self.pane_ms) - } -} - -/// How one output reads its definition: at evaluation time `T`, the pane -/// ending at `T - offset_ms`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PaneBinding { - pub offset_ms: i64, - /// The output's own `TimeRange` node: where its coverage's raw samples - /// feed when a stored output built in a `QueryRun` is built for it. - pub raw_input: u64, -} - -/// The stored summaries of one plan. -#[derive(Debug, Clone, PartialEq, Default)] -pub struct Sds { - /// Every definition, built in an `IngestionRun` or a `QueryRun`. - pub definitions: BTreeMap, - /// Every stored output, with its definition and pane. - pub outputs: BTreeMap, -} - -#[derive(Debug, thiserror::Error, PartialEq)] -pub enum SdsError { - #[error("node {0} is not in the DAG")] - UnknownNode(u64), - #[error("stored output {0}'s retention does not fit in i64 milliseconds")] - RetentionOverflow(u64), - /// Stored outputs read each other in a cycle. - #[error("stored outputs read each other in a cycle through {0:?}")] - Cycle(DefinitionKey), - /// An `IngestionRun` definition reads a `QueryRun` one, which is built - /// only when an evaluation needs it, so never in time for ingestion. - #[error("stored output built in an IngestionRun ({definition:?}) reads one built in a QueryRun ({input:?})")] - IngestionReadsQueryRun { - definition: DefinitionKey, - input: DefinitionKey, - }, - /// A stored output's coverage or inputs are not what the data plane can - /// build. - #[error("stored output {node}: {reason}")] - Unsupported { node: u64, reason: String }, -} - -impl Sds { - /// The definitions behind `plan`'s stored outputs, built in an - /// `IngestionRun` or in a `QueryRun`. - pub fn from_plan(plan: &InstallablePlan) -> Result { - let dag = &plan.dag.dag; - let nodes: HashMap = - dag.nodes.iter().map(|n| (n.id as u64, n)).collect(); - let mut sds = Sds::default(); - let mut stored_inputs: HashMap> = HashMap::new(); - for output in &plan.stored_outputs { - let node = output.node_id as u64; - let unsupported = |reason: &str| SdsError::Unsupported { - node, - reason: reason.into(), - }; - let output_schema = nodes - .get(&node) - .ok_or(SdsError::UnknownNode(node))? - .output_schema - .clone(); - let [selection] = output.coverage.selection.as_slice() else { - return Err(unsupported("its selection is not one box")); - }; - let (start, end) = relative_time(selection) - .ok_or_else(|| unsupported("its selection has no left-open time interval"))?; - let window = match output.inputs.raw_sample_windows.as_slice() { - [] => None, - [window] => Some(window.clone()), - _ => return Err(unsupported("it reads several raw-sample windows")), - }; - if window.is_none() && output.inputs.stored_outputs.is_empty() { - return Err(unsupported("it reads no input")); - } - let key = DefinitionKey { - id: DefinitionId(output.coverage.definition_id.clone()), - labels: label_constraints(selection), - pane_ms: end - start, - }; - let retention_ms = i64::try_from(output.retention_ms) - .map_err(|_| SdsError::RetentionOverflow(node))?; - let definition = - sds.definitions - .entry(key.clone()) - .or_insert_with(|| SummaryDefinition { - id: key.id.clone(), - labels: key.labels.clone(), - built_at: output.built_at, - pane_ms: key.pane_ms, - builder: node, - window: window.clone(), - output_schema, - stored_inputs: Vec::new(), - retention_ms, - }); - if output.built_at == StoredOutputBuild::IngestionRun { - definition.built_at = StoredOutputBuild::IngestionRun; - } - if node < definition.builder { - definition.builder = node; - definition.window = window.clone(); - } - definition.retention_ms = definition.retention_ms.max(retention_ms); - let binding = PaneBinding { - offset_ms: -end, - raw_input: window - .as_ref() - .map_or(node, |w| w.time_range_node_id as u64), - }; - sds.outputs.insert(node, (key, binding)); - stored_inputs.insert( - node, - output - .inputs - .stored_outputs - .iter() - .map(|n| *n as u64) - .collect(), - ); - } - sds.link_stored_inputs(&stored_inputs)?; - sds.ingestion_build_order()?; - Ok(sds) - } - - /// Records, for each definition, the definitions of the stored outputs - /// its builder reads, as the plan lists them (`StoredOutputInputs`). - fn link_stored_inputs(&mut self, inputs: &HashMap>) -> Result<(), SdsError> { - let stored: HashMap = self - .outputs - .iter() - .map(|(node, (key, _))| (*node, key.clone())) - .collect(); - for definition in self.definitions.values_mut() { - let mut found = Vec::new(); - for input in inputs.get(&definition.builder).into_iter().flatten() { - let key = stored.get(input).ok_or(SdsError::UnknownNode(*input))?; - if !found.contains(key) { - found.push(key.clone()); - } - } - found.sort(); - definition.stored_inputs = found; - } - Ok(()) - } - - /// The definitions built in ingestion runs, each after every definition - /// it reads (`stored_inputs`). Fails on a cycle, and when an - /// `IngestionRun` definition reads a `QueryRun` one. - pub fn ingestion_build_order(&self) -> Result, SdsError> { - let mut order = Vec::new(); - let mut state: HashMap<&DefinitionKey, bool> = HashMap::new(); // false: visiting - fn visit<'a>( - sds: &'a Sds, - id: &'a DefinitionKey, - state: &mut HashMap<&'a DefinitionKey, bool>, - order: &mut Vec, - ) -> Result<(), SdsError> { - match state.get(id) { - Some(true) => return Ok(()), - Some(false) => return Err(SdsError::Cycle(id.clone())), - None => {} - } - state.insert(id, false); - let definition = &sds.definitions[id]; - for input in &definition.stored_inputs { - if definition.built_at == StoredOutputBuild::IngestionRun - && sds.definitions[input].built_at == StoredOutputBuild::QueryRun - { - return Err(SdsError::IngestionReadsQueryRun { - definition: id.clone(), - input: input.clone(), - }); - } - visit(sds, input, state, order)?; - } - state.insert(id, true); - order.push(id.clone()); - Ok(()) - } - for id in self.definitions.keys() { - visit(self, id, &mut state, &mut order)?; - } - order.retain(|id| self.definitions[id].built_at == StoredOutputBuild::IngestionRun); - Ok(order) - } -} - -/// `(start, end]` of `selection`'s relative time, the offsets from the -/// evaluation time a pane covers, when it is a left-open interval. -fn relative_time(selection: &SelectionBox) -> Option<(i64, i64)> { - match selection.relative_time? { - (Bound::Excluded(start), Bound::Included(end)) if start < end => Some((start, end)), - _ => None, - } -} - -/// `selection`'s label constraints in the plan's JSON form; empty when it has -/// none. -fn label_constraints(selection: &SelectionBox) -> LabelConstraints { - if selection.columns.is_empty() { - return Arc::from(""); - } - let columns = SelectionBox { - columns: selection.columns.clone(), - relative_time: None, - }; - Arc::from(serde_json::to_string(&columns).expect("selections serialize")) -} - -/// The concrete selection of one summary instance: its definition's label -/// constraints and the time interval `(end_ms - pane_ms, end_ms]` the -/// plan's relative time binds to at evaluation time. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Selection { - pub labels: LabelConstraints, - pub pane_ms: i64, - pub end_ms: i64, -} - -impl Selection { - /// The pane `(end_ms - pane_ms, end_ms]`, with no label constraints. - pub fn ending_at(end_ms: i64, pane_ms: i64) -> Self { - Self { - labels: Arc::from(""), - pane_ms, - end_ms, - } - } -} - -impl DefinitionKey { - /// The concrete selection of this definition's pane ending at `end_ms`. - pub fn selection_ending_at(&self, end_ms: i64) -> Selection { - Selection { - labels: self.labels.clone(), - pane_ms: self.pane_ms, - end_ms, - } - } -} - -/// One stored summary: what it computes and which data it covers. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct SummaryInstanceKey { - pub definition: DefinitionId, - pub selection: Selection, -} - -impl SummaryInstanceKey { - pub fn new(definition: DefinitionId, selection: Selection) -> Self { - Self { - definition, - selection, - } - } -} - -/// Stored summary instances, by definition and selection. An instance is -/// stored once, when its coverage is complete, and never changes. -#[derive(Debug)] -pub struct SummaryStore { - instances: BTreeMap>, -} - -impl Default for SummaryStore { - fn default() -> Self { - Self::new() - } -} - -impl SummaryStore { - pub fn new() -> Self { - Self { - instances: BTreeMap::new(), - } - } - - pub fn insert(&mut self, key: SummaryInstanceKey, state: S) { - self.instances - .entry(key.definition) - .or_default() - .insert(key.selection, state); - } - - /// The instance of `definition` covering `selection`, if stored. Borrows - /// both, so a lookup allocates nothing. - pub fn get(&self, definition: &DefinitionId, selection: &Selection) -> Option<&S> { - self.instances.get(definition)?.get(selection) - } - - /// The instance of `definition`'s pane ending at `end_ms`, if stored. - pub fn get_pane(&self, definition: &DefinitionKey, end_ms: i64) -> Option<&S> { - self.get(&definition.id, &definition.selection_ending_at(end_ms)) - } - - /// Drops the instances of `definition` with `oldest`'s label constraints - /// and pane width whose pane ends before `oldest`'s. - pub fn evict_before(&mut self, definition: &DefinitionId, oldest: &Selection) { - let Some(instances) = self.instances.get_mut(definition) else { - return; - }; - let from = Selection { - end_ms: i64::MIN, - ..oldest.clone() - }; - let expired: Vec = instances - .range(from..oldest.clone()) - .map(|(selection, _)| selection.clone()) - .collect(); - for selection in expired { - instances.remove(&selection); - } - } - - pub fn len(&self) -> usize { - self.instances.values().map(BTreeMap::len).sum() - } - - pub fn is_empty(&self) -> bool { - self.instances.values().all(BTreeMap::is_empty) - } -} diff --git a/data_plane/src/store.rs b/data_plane/src/store.rs new file mode 100644 index 00000000..d91c3cea --- /dev/null +++ b/data_plane/src/store.rs @@ -0,0 +1,332 @@ +//! The data plane's time-indexed store, which the executor reads through +//! `asap_executor::sources::DataStore`: +//! +//! - **raw data**: every Remote Write sample of a metric some `Scan` of the +//! plan reads, as a time series table `(ts, value, *labels)`. A read +//! returns the samples in the request's time range; the executor's own +//! `TimeRange`s keep each window, an instant selector's latest sample per +//! series, and drop Prometheus stale markers. +//! - **materialized views**: each view's rows, appended by the ingestion +//! run that refreshed them with the `(start, end]` they cover, kept for +//! the view's `retention_ms`. These are held in the executor's +//! `InMemoryDataStore`, which reads exactly the rows tiling a request and +//! fails when one is missing. +//! +//! A run reads a [`RunStore`]: a snapshot that shares both parts, so the +//! engine appends between runs, never during one. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; + +use asap_executor::arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array, +}; +use asap_executor::planner::ir::operator::Source; +use asap_executor::planner::ir::schema::{DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; +use asap_executor::planner::plan::materialized_view::MaterializedView; +use asap_executor::runtime::{OutputStream, RunContext}; +use asap_executor::sources::{ + DataStore, InMemoryDataStore, MaterializedViewRequest, RawDataRequest, +}; +use asap_executor::values::{Batch, BatchColumn}; +use asap_executor::Error; +use futures::{stream, StreamExt}; + +use crate::series_ids::Series; + +/// Rows per raw-data batch a read returns. +const RAW_BATCH_ROWS: usize = 64 * 1024; + +/// One raw sample: its series (interned: its id and full label set, +/// `__name__` included), time and value. +#[derive(Debug, Clone, PartialEq)] +pub struct RawSample { + pub labels: Arc, + pub timestamp_ms: i64, + pub value: f64, +} + +/// The raw samples of each metric the plan scans, in arrival order. +#[derive(Debug, Default)] +pub struct RawData { + by_metric: HashMap>, +} + +impl RawData { + /// Keeps the samples of `metrics` from now on; samples of any other + /// metric are not stored ([`Self::reads`]). + pub fn for_metrics<'a>(metrics: impl IntoIterator) -> Self { + Self { + by_metric: metrics + .into_iter() + .map(|metric| (metric.clone(), Vec::new())) + .collect(), + } + } + + /// Whether some `Scan` reads `metric`. + pub fn reads(&self, metric: &str) -> bool { + self.by_metric.contains_key(metric) + } + + /// Appends `sample`; `false` (not stored) when no `Scan` reads its metric. + pub fn push(&mut self, sample: RawSample) -> bool { + match self.by_metric.get_mut(&*sample.labels.metric) { + Some(samples) => { + samples.push(sample); + true + } + None => false, + } + } + + /// Drops `metric`'s samples at or before `oldest_kept_ms`. + pub fn evict_through(&mut self, metric: &str, oldest_kept_ms: i64) { + if let Some(samples) = self.by_metric.get_mut(metric) { + if samples.iter().any(|s| s.timestamp_ms <= oldest_kept_ms) { + samples.retain(|s| s.timestamp_ms > oldest_kept_ms); + } + } + } + + /// Samples held, over every metric. + pub fn len(&self) -> usize { + self.by_metric.values().map(Vec::len).sum() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// `metric`'s samples in `time_range_ms` (`(start, end]`, every sample + /// when `None`) as batches of `request`'s schema. + fn read(&self, metric: &str, request: &RawDataRequest) -> Result, Error> { + let samples = self.by_metric.get(metric).map(Vec::as_slice).unwrap_or(&[]); + let in_range: Vec<&RawSample> = match request.time_range_ms { + Some((start, end)) => samples + .iter() + .filter(|s| start < s.timestamp_ms && s.timestamp_ms <= end) + .collect(), + None => samples.iter().collect(), + }; + let columns: Vec = request + .schema + .fields + .iter() + .enumerate() + .map(|(index, field)| { + if Some(index) == request.schema.time_index { + Ok(ColumnKind::Time) + } else if field.name == PROMQL_SERIES_IDENTITY { + Ok(ColumnKind::SeriesIdentity) + } else if field.name == "value" + && field.dtype == FieldDataType::Plain(DataType::Float64) + { + Ok(ColumnKind::Value) + } else if field.dtype == FieldDataType::Plain(DataType::Utf8) { + Ok(ColumnKind::Label(field.name.clone())) + } else { + Err(Error::Invalid(format!( + "time series {metric} has no column {} of type {:?}", + field.name, field.dtype + ))) + } + }) + .collect::>()?; + in_range + .chunks(RAW_BATCH_ROWS) + .map(|rows| { + let columns = columns + .iter() + .map(|kind| kind.column(rows).map(BatchColumn::Plain)) + .collect::, Error>>()?; + Batch::try_from_columns(request.schema.clone(), columns) + }) + .collect() + } +} + +/// What one column of a time series table holds. +enum ColumnKind { + Time, + Value, + /// The canonical JSON of every label, `__name__` included. + SeriesIdentity, + /// One label's value, empty when the series lacks it. + Label(String), +} + +impl ColumnKind { + fn column(&self, rows: &[&RawSample]) -> Result { + Ok(match self { + Self::Time => Arc::new(TimestampMillisecondArray::from_iter_values( + rows.iter().map(|s| s.timestamp_ms), + )), + Self::Value => Arc::new(Float64Array::from_iter_values(rows.iter().map(|s| s.value))), + Self::SeriesIdentity => series_text_column(rows, |series| { + serde_json::to_string(&**series).expect("labels serialize") + })?, + Self::Label(name) => { + series_text_column(rows, |series| series.get(name).cloned().unwrap_or_default())? + } + }) + } +} + +/// A text column computed once per series of `rows`, as the batch layout +/// holds text: a dictionary of the distinct texts and each row's code. +fn series_text_column( + rows: &[&RawSample], + text_of: impl Fn(&Series) -> String, +) -> Result { + let mut code_of_series: HashMap = HashMap::new(); + let mut code_of_text: HashMap = HashMap::new(); + let mut texts: Vec = Vec::new(); + let keys: UInt32Array = rows + .iter() + .map(|sample| { + *code_of_series.entry(sample.labels.id).or_insert_with(|| { + let text = text_of(&sample.labels); + *code_of_text.entry(text.clone()).or_insert_with(|| { + texts.push(text); + texts.len() as u32 - 1 + }) + }) + }) + .collect(); + let dictionary = DictionaryArray::try_new(keys, Arc::new(StringArray::from(texts))) + .map_err(|e| Error::Invalid(e.to_string()))?; + Ok(Arc::new(dictionary)) +} + +/// What one run reads: the store as it was when the run started. It +/// records the first view read that found no rows, which says the query +/// cannot be answered from what is stored (rather than that it failed). +pub struct RunStore { + raw_data: Arc, + views: Arc, + missing_view_rows: Mutex>, +} + +impl RunStore { + pub fn new(raw_data: Arc, views: Arc) -> Self { + Self { + raw_data, + views, + missing_view_rows: Mutex::new(None), + } + } + + /// Why the run's first view read failed, if one did. + pub fn missing_view_rows(&self) -> Option { + self.missing_view_rows + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } +} + +impl DataStore for RunStore { + fn read_raw_data( + &self, + source: &Source, + request: &RawDataRequest, + context: RunContext, + ) -> Result, Error> { + let Source::TimeSeries { metric } = source else { + return Err(Error::Invalid(format!( + "the data plane stores time series only, not {source:?}" + ))); + }; + let batches = self.raw_data.read(metric, request)?; + Ok(stream::iter(batches) + .map(move |batch| { + if context.is_cancelled() { + return Err(Error::Cancelled); + } + let _reservation = context.reserve(batch.memory_bytes())?; + Ok(batch) + }) + .boxed_local()) + } + + fn read_materialized_view( + &self, + view: &MaterializedView, + request: &MaterializedViewRequest, + context: RunContext, + ) -> Result, Error> { + self.views + .read_materialized_view(view, request, context) + .inspect_err(|error| { + self.missing_view_rows + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get_or_insert_with(|| error.to_string()); + }) + } +} + +/// The engine's store: raw data and materialized views, each shared with +/// the [`RunStore`] of the run in progress and changed only between runs. +#[derive(Default)] +pub struct TimeIndexedStore { + raw_data: Arc, + views: Arc, +} + +impl TimeIndexedStore { + /// An empty store keeping the samples of `metrics`. + pub fn for_metrics<'a>(metrics: impl IntoIterator) -> Self { + Self { + raw_data: Arc::new(RawData::for_metrics(metrics)), + views: Arc::default(), + } + } + + /// The snapshot one run reads. + pub fn run_store(&self) -> Arc { + Arc::new(RunStore::new(self.raw_data.clone(), self.views.clone())) + } + + pub fn raw_data(&self) -> &RawData { + &self.raw_data + } + + /// The raw data, to append or evict; no run may be in progress. + pub fn raw_data_mut(&mut self) -> &mut RawData { + Arc::get_mut(&mut self.raw_data).expect("no run reads the store while it changes") + } + + /// Appends `batches` as `view`'s rows covering `time_range_ms`; no run + /// may be in progress. + pub fn append_view_rows( + &mut self, + view: &MaterializedView, + time_range_ms: (i64, i64), + batches: Vec, + ) -> Result<(), Error> { + Arc::get_mut(&mut self.views) + .expect("no run reads the store while it changes") + .append_materialized_view(view, time_range_ms, batches) + } +} + +/// Every metric a `Scan` of `nodes` reads. +pub fn scanned_metrics<'a>( + nodes: impl IntoIterator< + Item = &'a asap_executor::planner::plan::physical_asap_dag::PhysicalASAPDAGNode, + >, +) -> BTreeSet { + use asap_executor::planner::ir::{NonASAPOp, Operator}; + nodes + .into_iter() + .filter_map(|node| match &node.payload { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric }, + .. + }) => Some(metric.clone()), + _ => None, + }) + .collect() +} diff --git a/data_plane/tests/answers.rs b/data_plane/tests/answers.rs deleted file mode 100644 index c6ac3a27..00000000 --- a/data_plane/tests/answers.rs +++ /dev/null @@ -1,495 +0,0 @@ -//! The data plane answers planned instant queries over HTTP, from stored -//! panes and from raw samples it fetches, and forwards what it cannot answer. - -mod common; - -use std::collections::BTreeMap; -use std::sync::Arc; - -use axum::body::Body; -use axum::extract::Query; -use axum::http::{HeaderMap, Request, StatusCode}; -use axum::routing::get; -use axum::{Json, Router}; -use data_plane::remote_write::{encode, Label, Sample, TimeSeries, WriteRequest}; -use data_plane::{router, DataPlane, EXECUTION_HEADER, FORWARD_REASON_HEADER}; -use http_body_util::BodyExt; -use serde_json::{json, Value}; -use tower::ServiceExt; - -const MINUTE: i64 = 60_000; - -/// `(labels, timestamp ms, value)`, labels including `__name__`. -type Point = (BTreeMap, i64, f64); - -fn series(metric: &str, labels: &[(&str, &str)]) -> BTreeMap { - let mut map = BTreeMap::from([("__name__".to_string(), metric.to_string())]); - map.extend(labels.iter().map(|(k, v)| (k.to_string(), v.to_string()))); - map -} - -/// A stand-in Prometheus holding `points`. It answers -/// `{__name__="m"}[Nms]` at `time` with the samples in `[time - N, time]`, -/// both ends included as Prometheus 2 does, and anything else with -/// `{"forwarded": query}`. -async fn fake_prometheus(points: Vec) -> String { - fake_prometheus_requiring(points, &[]).await -} - -/// [`fake_prometheus`] that answers 401 to a request missing any of -/// `required` (header name, value). -async fn fake_prometheus_requiring(points: Vec, required: &[(&str, &str)]) -> String { - let points = Arc::new(points); - let required: Arc> = Arc::new( - required - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - ); - let handler = move |headers: HeaderMap, Query(params): Query>| { - let points = points.clone(); - let required = required.clone(); - async move { - let authorized = required.iter().all(|(name, value)| { - headers - .get(name.as_str()) - .is_some_and(|v| v == value.as_str()) - }); - if !authorized { - return (StatusCode::UNAUTHORIZED, Json(json!({"status": "error"}))); - } - let query = ¶ms["query"]; - let parsed = query - .strip_prefix("{__name__=\"") - .and_then(|rest| rest.split_once("\"}[")) - .and_then(|(metric, rest)| { - Some((metric, rest.strip_suffix("ms]")?.parse::().ok()?)) - }); - let Some((metric, range)) = parsed else { - return (StatusCode::OK, Json(json!({ "forwarded": query }))); - }; - let time = (params["time"].parse::().unwrap() * 1000.0).round() as i64; - let mut by_series: BTreeMap<&BTreeMap, Vec> = BTreeMap::new(); - for (labels, t, v) in points.iter() { - if labels["__name__"] == metric && time - range <= *t && *t <= time { - by_series - .entry(labels) - .or_default() - .push(json!([*t as f64 / 1000.0, v.to_string()])); - } - } - let result: Vec = by_series - .into_iter() - .map(|(labels, values)| json!({"metric": labels, "values": values})) - .collect(); - ( - StatusCode::OK, - Json( - json!({"status": "success", "data": {"resultType": "matrix", "result": result}}), - ), - ) - } - }; - let app = Router::new().route("/api/v1/query", get(handler)); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - url -} - -struct Response { - execution: Option, - reason: Option, - body: Value, -} - -async fn instant(app: &Router, query: &str, time_ms: i64) -> Response { - let uri = format!( - "/api/v1/query?{}", - form_urlencoded::Serializer::new(String::new()) - .append_pair("query", query) - .append_pair("time", &(time_ms as f64 / 1000.0).to_string()) - .finish() - ); - let response = app - .clone() - .oneshot(Request::get(uri).body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - let header = |name| { - response - .headers() - .get(name) - .map(|v: &axum::http::HeaderValue| v.to_str().unwrap().to_string()) - }; - let (execution, reason) = (header(EXECUTION_HEADER), header(FORWARD_REASON_HEADER)); - let body = response.into_body().collect().await.unwrap().to_bytes(); - Response { - execution, - reason, - body: serde_json::from_slice(&body).unwrap(), - } -} - -/// `(job, value)` of each result series, by job. -fn by_job(body: &Value) -> Vec<(String, f64)> { - let mut rows: Vec<(String, f64)> = body["data"]["result"] - .as_array() - .unwrap() - .iter() - .map(|s| { - ( - s["metric"]["job"].as_str().unwrap().to_string(), - s["value"][1].as_str().unwrap().parse().unwrap(), - ) - }) - .collect(); - rows.sort_by(|a, b| a.0.cmp(&b.0)); - rows -} - -/// Example 1 runs at query time: the data plane fetches the last minute of -/// raw samples from Prometheus and runs the plan over them. A sample at the -/// range's start, which Prometheus 2 returns, is excluded as in PromQL. -#[tokio::test] -async fn answers_query_time_plans_from_raw_samples() { - let mut points = Vec::new(); - for (job, instance) in [("a", "1"), ("a", "2"), ("b", "1")] { - let labels = series( - "http_requests_total", - &[("job", job), ("instance", instance)], - ); - for t in (0..10 * MINUTE).step_by(15_000) { - points.push((labels.clone(), t, t as f64 / 1000.0)); - } - } - let state = DataPlane::new( - common::plan_json("example1", true), - fake_prometheus(points).await, - 0, - ) - .unwrap(); - let app = router(Arc::new(state)); - let response = instant( - &app, - "sum by (job) (rate(http_requests_total[1m]))", - 5 * MINUTE, - ) - .await; - assert_eq!( - response.execution.as_deref(), - Some("plan"), - "{:?}", - response.body - ); - let result = by_job(&response.body); - let jobs: Vec<&str> = result.iter().map(|(job, _)| job.as_str()).collect(); - assert_eq!(jobs, ["a", "b"]); - for ((_, value), expected) in result.iter().zip([2.0, 1.0]) { - assert!((value - expected).abs() < 1e-9, "{result:?}"); - } - // (4 min, 5 min] holds the samples at 4:15 .. 5:00: 255 + 270 + 285 + 300. - // Including the one at 4:00 would add 240. - let response = instant( - &app, - "topk by (job) (10, sum_over_time(http_requests_total[1m]))", - 5 * MINUTE, - ) - .await; - assert_eq!( - response.execution.as_deref(), - Some("plan"), - "{:?}", - response.body - ); - let result = by_job(&response.body); - assert_eq!(result.len(), 3, "{result:?}"); - assert!( - result.iter().all(|(_, value)| *value == 1110.0), - "{result:?}" - ); - - let response = instant(&app, "up", 5 * MINUTE).await; - assert_eq!(response.execution.as_deref(), Some("forwarded")); - assert_eq!( - response.reason, None, - "unplanned queries are simply forwarded" - ); - assert_eq!(response.body, json!({"forwarded": "up"})); -} - -/// Example 4b reads six stored 10-minute KLL panes built from Remote Write; -/// at a time off the pane grid it forwards and says why. -#[tokio::test] -async fn answers_from_panes_built_from_remote_write() { - let state = Arc::new( - DataPlane::new( - common::plan_json("example4b", false), - fake_prometheus(vec![]).await, - 0, - ) - .unwrap(), - ); - let app = router(state.clone()); - let timeseries = ["a", "b"] - .into_iter() - .enumerate() - .map(|(i, job)| TimeSeries { - labels: vec![ - Label { - name: "__name__".into(), - value: "latency_ms".into(), - }, - Label { - name: "job".into(), - value: job.into(), - }, - ], - samples: (MINUTE..75 * MINUTE) - .step_by(15_000) - .map(|t| Sample { - timestamp: t, - value: (i * 1000) as f64 + ((t / 15_000) % 240) as f64, - }) - .collect(), - }) - .collect(); - let write = app - .clone() - .oneshot( - Request::post("/api/v1/write") - .body(Body::from(encode(&WriteRequest { timeseries }))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(write.status(), StatusCode::NO_CONTENT); - // Panes ending at 20..70 min; the first, (0, 10 min], is skipped. - for _ in 0..500 { - if state.engine.queued_samples() == 0 { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - assert_eq!(state.engine.stats().panes_built, 6); - - let query = "quantile_over_time(0.99, latency_ms[60m])"; - let response = instant(&app, query, 70 * MINUTE).await; - assert_eq!( - response.execution.as_deref(), - Some("plan"), - "{:?}", - response.body - ); - let result = by_job(&response.body); - assert_eq!(result.len(), 2); - for ((_, value), base) in result.iter().zip([0.0, 1000.0]) { - assert!( - *value >= base + 236.0 && *value <= base + 239.0, - "{result:?}" - ); - } - - let response = instant(&app, query, 70 * MINUTE + 1_000).await; - assert_eq!(response.execution.as_deref(), Some("forwarded")); - assert!(response.reason.unwrap().contains("pane grid")); -} - -/// A planned query's raw-range reads carry the caller's auth and tenancy -/// headers, as a forwarded query does: a Prometheus that requires them -/// serves the read, and the plan answers. -#[tokio::test] -async fn raw_range_reads_forward_auth_and_tenancy_headers() { - let labels = series("http_requests_total", &[("job", "a"), ("instance", "1")]); - let points = (0..10 * MINUTE) - .step_by(15_000) - .map(|t| (labels.clone(), t, t as f64 / 1000.0)) - .collect(); - let required = [ - ("authorization", "Bearer secret"), - ("x-scope-orgid", "tenant-a"), - ]; - let state = DataPlane::new( - common::plan_json("example1", true), - fake_prometheus_requiring(points, &required).await, - 0, - ) - .unwrap(); - let app = router(Arc::new(state)); - let uri = format!( - "/api/v1/query?{}", - form_urlencoded::Serializer::new(String::new()) - .append_pair("query", "sum by (job) (rate(http_requests_total[1m]))") - .append_pair("time", "300") - .finish() - ); - let request = |with_headers: bool| { - let mut request = Request::get(uri.clone()); - if with_headers { - request = request - .header("authorization", "Bearer secret") - .header("x-scope-orgid", "tenant-a"); - } - request.body(Body::empty()).unwrap() - }; - let response = app.clone().oneshot(request(true)).await.unwrap(); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response.headers().get(EXECUTION_HEADER).unwrap(), - "plan", - "the raw read was authorized" - ); - // Without them the read is refused, so the query is forwarded, and - // Prometheus refuses that too. - let response = app.clone().oneshot(request(false)).await.unwrap(); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert_eq!( - response.headers().get(EXECUTION_HEADER).unwrap(), - "forwarded" - ); - assert!(response - .headers() - .get(FORWARD_REASON_HEADER) - .unwrap() - .to_str() - .unwrap() - .contains("fetching raw samples")); -} - -/// A planned query at a time Prometheus rejects (not finite, or beyond the -/// millisecond range) gets 400 like Prometheus, and the engine keeps -/// running. -#[tokio::test] -async fn rejects_evaluation_times_out_of_range() { - let state = Arc::new( - DataPlane::new( - common::plan_json("example4b", true), - fake_prometheus(vec![]).await, - 0, - ) - .unwrap(), - ); - let app = router(state.clone()); - for time in ["-1e300", "1e300", "NaN", "inf"] { - let uri = format!( - "/api/v1/query?{}", - form_urlencoded::Serializer::new(String::new()) - .append_pair("query", "quantile_over_time(0.99, latency_ms[60m])") - .append_pair("time", time) - .finish() - ); - let response = app - .clone() - .oneshot(Request::get(uri).body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST, "time={time}"); - let body = response.into_body().collect().await.unwrap().to_bytes(); - let body: Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(body["errorType"], "bad_data", "time={time}"); - } - assert!(state.engine.is_running()); -} - -/// After a producer's event-time barrier, the last pane of a series that -/// stopped sending is closed, and the planned query is served from the -/// plan instead of forwarded. Input stays open. -#[tokio::test] -async fn barrier_serves_the_window_of_a_stopped_series() { - let state = Arc::new( - DataPlane::new( - common::plan_json("example4b", false), - fake_prometheus(vec![]).await, - 15_000, - ) - .unwrap(), - ); - let app = router(state.clone()); - let write = |from: i64, to: i64| { - let request = WriteRequest { - timeseries: vec![TimeSeries { - labels: vec![ - Label { - name: "__name__".into(), - value: "latency_ms".into(), - }, - Label { - name: "job".into(), - value: "a".into(), - }, - ], - samples: (from..to) - .step_by(15_000) - .map(|t| Sample { - timestamp: t, - value: (t / 15_000 % 240) as f64, - }) - .collect(), - }], - }; - Request::post("/api/v1/write") - .body(Body::from(encode(&request))) - .unwrap() - }; - let written = app - .clone() - .oneshot(write(MINUTE, 70 * MINUTE)) - .await - .unwrap(); - assert_eq!(written.status(), StatusCode::NO_CONTENT); - let query = "quantile_over_time(0.99, latency_ms[60m])"; - // The series stopped at 69:45; with 15 s lateness (60, 70 min] is open. - let response = instant(&app, query, 70 * MINUTE).await; - assert_eq!(response.execution.as_deref(), Some("forwarded")); - - let barrier = app - .clone() - .oneshot( - Request::post(format!( - "/api/v1/precompute/watermark?time_ms={}", - 70 * MINUTE - )) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(barrier.status(), StatusCode::OK); - let body = barrier.into_body().collect().await.unwrap().to_bytes(); - let body: Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(body["barrier_ms"], 70 * MINUTE, "{body}"); - let response = instant(&app, query, 70 * MINUTE).await; - assert_eq!( - response.execution.as_deref(), - Some("plan"), - "{:?}", - response.body - ); - - let written = app - .clone() - .oneshot(write(70 * MINUTE + 15_000, 72 * MINUTE)) - .await - .unwrap(); - assert_eq!(written.status(), StatusCode::NO_CONTENT, "input stays open"); - for uri in [ - "/api/v1/precompute/watermark", - "/api/v1/precompute/watermark?time_ms=soon", - "/api/v1/precompute/watermark?time_ms=1.5", - ] { - let bad = app - .clone() - .oneshot(Request::post(uri).body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(bad.status(), StatusCode::BAD_REQUEST, "{uri}"); - let body = bad.into_body().collect().await.unwrap().to_bytes(); - let body: Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(body["errorType"], "bad_data", "{uri}"); - assert!( - body["error"].as_str().unwrap().contains("time_ms"), - "{uri}: {body}" - ); - } -} diff --git a/data_plane/tests/common/mod.rs b/data_plane/tests/common/mod.rs index a7b7b434..a58ccfb2 100644 --- a/data_plane/tests/common/mod.rs +++ b/data_plane/tests/common/mod.rs @@ -22,7 +22,9 @@ pub fn workload(example: &str) -> PlanningWorkload { /// keeps raw data or not. #[allow(dead_code)] pub fn plan(example: &str, raw_data_retained: bool) -> InstallablePlan { - plan_workload(&workload(example), &inputs(raw_data_retained)).unwrap() + plan_workload(&workload(example), &inputs(raw_data_retained)) + .unwrap() + .0 } #[allow(dead_code)] diff --git a/data_plane/tests/no_forward.rs b/data_plane/tests/no_forward.rs index 4c4c1a1b..c240a444 100644 --- a/data_plane/tests/no_forward.rs +++ b/data_plane/tests/no_forward.rs @@ -72,7 +72,7 @@ fn instant(query: &str, time_ms: i64) -> String { ) } -/// With the flag, an unplanned query, an evaluation whose pane is missing +/// With the flag, an unplanned query, an evaluation whose windows are missing /// and a range query each get a Prometheus `execution` error with the /// reason, and the stand-in Prometheus receives no request. #[tokio::test] @@ -92,13 +92,13 @@ async fn refuses_what_the_plan_cannot_answer_without_asking_prometheus() { "{body}" ); - // Nothing was written, so the panes the query reads are not stored. + // Nothing was written, so the windows the query reads are not stored. let query = "quantile_over_time(0.99, latency_ms[60m])"; let (status, execution, body) = get(&app, &instant(query, 70 * MINUTE)).await; assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); assert_eq!(execution.as_deref(), Some("refused")); assert!( - body["error"].as_str().unwrap().contains("not stored"), + body["error"].as_str().unwrap().contains("no rows tiling"), "{body}" ); @@ -131,19 +131,16 @@ async fn forwards_without_the_flag() { #[test] fn refuses_to_install_a_plan_with_a_query_it_cannot_run() { let mut plan = common::plan("example1", true); - // A label matcher on a column the `Scan` does not have: the selector of - // its raw-sample window does not compile, so the queries that read it - // cannot run. - let scan = plan.query_time_raw_windows[0].scan_node_id; + // A predicate on a column the `Scan` does not have: the queries that + // read it do not compile. let node = plan - .dag .dag .nodes .iter_mut() - .find(|n| n.id == scan) + .find(|n| matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) .unwrap(); let Operator::NonASAP(NonASAPOp::Scan { predicates, .. }) = &mut node.payload else { - panic!("not a Scan"); + unreachable!(); }; predicates.push(Predicate(ScalarExpr::Column(99))); plan.validate().unwrap(); diff --git a/data_plane/tests/per_sample.rs b/data_plane/tests/per_sample.rs deleted file mode 100644 index 45330585..00000000 --- a/data_plane/tests/per_sample.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! Ingestion updates each open pane's summary state per sample, through the -//! executor's `StateColumnUpdater`, and builds the pane from columns; the -//! panes it stores equal the plan's own build over the same samples. - -mod common; - -use std::collections::BTreeMap; - -use asap_executor::planner::ir::schema::{FieldDataType, SketchStatistic}; -use asap_executor::summary_kernels::exact::ExactAccumulator; -use asap_executor::values::Value; -use data_plane::precompute::{Precompute, PrecomputeOptions, RawSample, SummaryInstance}; -use data_plane::sds::Sds; - -const MINUTE: i64 = 60_000; - -fn options() -> PrecomputeOptions { - PrecomputeOptions { - allowed_lateness_ms: 0, - ..PrecomputeOptions::default() - } -} - -/// Samples every 15 s in `[from, to)` of `metric` for each `job`; values -/// grow with time and differ by job. -fn samples(metric: &str, jobs: usize, from: i64, to: i64) -> Vec { - let series: Vec<_> = (0..jobs) - .map(|j| { - data_plane::series_ids::intern(&BTreeMap::from([ - ("__name__".to_string(), metric.to_string()), - ("job".to_string(), format!("job{j}")), - ("instance".to_string(), format!("{}", j % 3)), - ])) - }) - .collect(); - (from..to) - .step_by(15_000) - .flat_map(|t| { - series.iter().enumerate().map(move |(j, labels)| RawSample { - labels: labels.clone(), - timestamp_ms: t, - value: (j * 1000) as f64 + (t / 15_000 % 97) as f64 + t as f64 / 1e6, - }) - }) - .collect() -} - -/// The state in `value`, as comparable data: an exact accumulator's -/// serialized bytes, or a quantile sketch's quantiles at every percent. -/// KLL seeds its compaction randomly per sketch, so two builds of the same -/// samples agree on content, not on bytes; below k they hold every sample. -fn state_of(value: &Value) -> Vec { - let Value::Summary { family, state } = value else { - panic!("not a state: {value:?}"); - }; - if let Some(exact) = state.as_any().downcast_ref::() { - return serde_json::to_vec(exact).unwrap(); - } - assert!(matches!(family, FieldDataType::Sketch(..)), "{family:?}"); - (1..100) - .flat_map(|p| { - let quantile = SketchStatistic::Quantile { - q: p as f64 / 100.0, - }; - state.estimate(&quantile).unwrap().to_le_bytes() - }) - .collect() -} - -/// Every cell of `pane` as comparable data, its rows sorted. (Tests read -/// rows to compare them; the data plane does not.) -fn cells(pane: &SummaryInstance) -> Vec>> { - let mut rows: Vec>> = pane - .to_rows() - .into_iter() - .map(|row| { - row.iter() - .map(|value| match value { - Value::Summary { .. } => state_of(value), - other => other.key().unwrap(), - }) - .collect() - }) - .collect(); - rows.sort(); - rows -} - -/// For Example 1's counter rates and sums over 10 s panes, Example 3b's -/// 1-minute KLL panes and Example 4b's 10-minute KLL panes, on a deployment -/// without raw data: every pane built per sample equals the executor's batch build of the same samples, row by row. -#[test] -fn per_sample_panes_equal_the_batch_build() { - for (example, metric, to) in [ - ("example1", "http_requests_total", 12 * MINUTE), - ("example3b", "latency_ms", 12 * MINUTE), - ("example4b", "latency_ms", 75 * MINUTE), - ] { - let plan = common::plan(example, false); - let sds = Sds::from_plan(&plan).unwrap(); - let all = samples(metric, 7, MINUTE, to + 1); - let mut batch = Precompute::batch(&plan.dag.dag, &sds, options()).unwrap(); - batch.ingest(&all); - assert_eq!(batch.per_sample_definitions(), 0); - { - let mut per_sample = Precompute::new(plan.dag.dag.clone(), &sds, options()).unwrap(); - assert_eq!( - per_sample.per_sample_definitions(), - sds.definitions.len(), - "{example}: every definition is built per sample" - ); - per_sample.ingest(&all); - assert_eq!(per_sample.stats, batch.stats, "{example}"); - let mut compared = 0; - for definition in sds.definitions.values() { - let mut end = 0; - while end <= to { - let selection = definition.selection_ending_at(end); - let expected = batch.store.get(&definition.id, &selection); - let actual = per_sample.store.get(&definition.id, &selection); - assert_eq!(actual.is_some(), expected.is_some(), "{example} pane {end}"); - if let (Some(actual), Some(expected)) = (actual, expected) { - for batch in actual.batches.iter().chain(&expected.batches) { - assert_eq!(**batch.schema(), definition.output_schema); - } - assert_eq!(cells(actual), cells(expected), "{example} pane {end}"); - compared += usize::from(expected.num_rows() > 0); - } - end += definition.pane_ms; - } - } - assert!(compared > 0, "{example}: no pane compared"); - } - } -} - -/// The plan's label matchers apply per sample as in the batch build: only -/// the selected series get a state. -#[test] -fn per_sample_panes_apply_label_matchers() { - let mut workload = common::workload("example4b"); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = - asap_executor::planner::workload::Query( - "quantile_over_time(0.99, latency_ms{instance=\"1\"}[60m])".into(), - ); - let plan = control_plane::plan_workload(&workload, &common::inputs(false)).unwrap(); - let sds = Sds::from_plan(&plan).unwrap(); - let id = sds.definitions.keys().next().unwrap().clone(); - let all = samples("latency_ms", 7, MINUTE, 25 * MINUTE); - let mut per_sample = Precompute::new(plan.dag.dag.clone(), &sds, options()).unwrap(); - assert_eq!(per_sample.per_sample_definitions(), 1); - per_sample.ingest(&all); - let mut batch = Precompute::batch(&plan.dag.dag, &sds, options()).unwrap(); - batch.ingest(&all); - let (actual, expected) = ( - per_sample.store.get_pane(&id, 20 * MINUTE).unwrap(), - batch.store.get_pane(&id, 20 * MINUTE).unwrap(), - ); - assert_eq!(cells(expected).len(), 2, "jobs 1 and 4 are instance 1"); - assert_eq!(cells(actual), cells(expected)); -} - -/// A resent sample (a Remote Write retry, for instance) is not -/// after its series' previous sample: it is dropped and counted, and the -/// pane is built as without it, instead of failing. -#[test] -fn a_resent_sample_is_dropped_not_failing_or_doubling_the_pane() { - let plan = common::plan("example1", false); - let sds = Sds::from_plan(&plan).unwrap(); - let all = samples("http_requests_total", 3, MINUTE, 3 * MINUTE + 1); - let mut resent = all.clone(); - let again = resent[10].clone(); - resent.insert(11, again); - let mut batch = Precompute::batch(&plan.dag.dag, &sds, options()).unwrap(); - batch.ingest(&all); - let mut per_sample = Precompute::new(plan.dag.dag.clone(), &sds, options()).unwrap(); - assert_eq!(per_sample.per_sample_definitions(), sds.definitions.len()); - per_sample.ingest(&resent); - assert_eq!(per_sample.stats.samples_dropped_duplicate, 1); - assert_eq!(per_sample.stats.panes_failed, 0); - assert_eq!(per_sample.stats.panes_built, batch.stats.panes_built); - for definition in sds.definitions.values() { - let mut end = 0; - while end <= 3 * MINUTE { - let selection = definition.selection_ending_at(end); - let expected = batch.store.get(&definition.id, &selection); - let actual = per_sample.store.get(&definition.id, &selection); - assert_eq!( - actual.map(cells), - expected.map(cells), - "pane ending at {end}" - ); - end += definition.pane_ms; - } - } -} diff --git a/data_plane/tests/planner_executor_answers.rs b/data_plane/tests/planner_executor_answers.rs index c9938490..ae8c031b 100644 --- a/data_plane/tests/planner_executor_answers.rs +++ b/data_plane/tests/planner_executor_answers.rs @@ -47,14 +47,15 @@ fn plan_without_raw_samples( ignore_accuracy_guarantees: bool, ) -> InstallablePlan { let mut inputs = common::inputs(false); - inputs.plan_selection.ignore_accuracy_guarantees = ignore_accuracy_guarantees; - plan_workload(workload, &inputs).unwrap_or_else(|e| panic!("{e}")) + inputs.ignore_accuracy_guarantees = ignore_accuracy_guarantees; + plan_workload(workload, &inputs) + .unwrap_or_else(|e| panic!("{e}")) + .0 } -/// A data plane running `plan` with `--no-forward`, its panes built from -/// `points` written by Remote Write, one request per second, and closed by -/// the barrier at -/// `EVALUATION_MS`. +/// A data plane running `plan` with `--no-forward`, its windows refreshed +/// from `points` written by Remote Write, one request per second, and +/// closed by the barrier at `EVALUATION_MS`. async fn run(plan: &InstallablePlan, points: &[Point]) -> (Router, Arc) { let options = EngineOptions { no_forward: true, @@ -70,9 +71,16 @@ async fn run(plan: &InstallablePlan, points: &[Point]) -> (Router, Arc>> = BTreeMap::new(); for point in points { by_second @@ -119,19 +127,21 @@ async fn run(plan: &InstallablePlan, points: &[Point]) -> (Router, Arc PlanningWorkload { /// [`p99_workload`] with the quantile `q`. fn quantile_workload(outer: &str, q: f64, values_non_negative: bool) -> PlanningWorkload { + let sign = if values_non_negative { + "non_negative" + } else { + "unknown" + }; serde_json::from_str(&format!( r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ "query": "{outer} by (region, service, job) (quantile_over_time({q}, data[100s]))", - "demand": {{"fixed_interval": 1000}}, + "demand": {{"fixed_interval_at": {{"interval": 1000, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", - "sample_interval": 100, "values_non_negative": {values_non_negative}}}}}}}}}"# + "sample_interval": 100, "column_signs": {{"value": "{sign}"}}}}}}}}}}"# )) .unwrap() } @@ -263,11 +278,24 @@ fn interpolated_quantile(q: f64, window: &[f64]) -> f64 { sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) } -/// Every group's answer is within the relative error 0.05 of `combine` of -/// its series' exact p99s. +/// The relative error α of the plan's DDSketch, the largest when several. +fn ddsketch_alpha(plan: &InstallablePlan) -> f64 { + plan.dag + .nodes + .iter() + .filter_map(|node| match node.parameters { + Some(asap_planner::SketchParams::DDSketch { alpha }) => Some(alpha), + _ => None, + }) + .fold(0.0, f64::max) +} + +/// Every group's answer is within the relative error `alpha` of `combine` +/// of its series' exact p99s. fn assert_p99s_within_alpha( answers: &[(Labels, f64)], windows: &Windows, + alpha: f64, combine: impl Fn(&[f64]) -> f64, ) { assert_eq!(answers.len(), windows.len(), "{answers:?}"); @@ -283,7 +311,7 @@ fn assert_p99s_within_alpha( .collect(); let exact = combine(&p99s); assert!( - (value - exact).abs() <= 0.05 * exact.abs(), + (value - exact).abs() <= alpha * exact.abs(), "{labels:?}: {value} vs exact {exact}" ); } @@ -298,26 +326,34 @@ fn average(values: &[f64]) -> f64 { /// holds a third zeros, and every group's average of p99s is within 0.05 of /// the exact average of PromQL's interpolated p99s. #[tokio::test] -async fn ddsketch_panes_over_a_gauge_reporting_zero_answer_within_alpha() { +async fn ddsketch_windows_over_a_gauge_reporting_zero_answer_within_alpha() { let query = "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))"; let plan = plan_without_raw_samples(&p99_workload("avg", true), false); - assert!(!plan.stored_outputs.is_empty(), "panes built at ingestion"); + assert!( + !plan.materialized_views().unwrap().is_empty(), + "tumbling windows kept at ingestion" + ); let (points, windows) = data_samples(8, 1_100, 100, |s, step| { (s + 1) as f64 * [0.0, 3.0, 5.0][(step % 3) as usize] }); let (app, _) = run(&plan, &points).await; - assert_p99s_within_alpha(&answer(&app, query).await, &windows, average); + assert_p99s_within_alpha(&answer(&app, query).await, &windows, 0.05, average); } /// Signed samples (negative, zero and positive) are summarized by the -/// selected DDSketch panes through their negative store and zero bucket: +/// selected DDSketch windows through their negative store and zero bucket: /// the `max` of the per-series p99s is within 0.05 of the exact value, -/// including groups whose samples are all negative. +/// including groups whose samples are all negative. A gauge of unknown +/// sign has no plan with an accuracy guarantee without raw samples, so it +/// is planned with accuracy guarantees ignored. #[tokio::test] -async fn ddsketch_panes_over_signed_samples_answer_within_alpha() { +async fn ddsketch_windows_over_signed_samples_answer_within_alpha() { let query = "max by (region, service, job) (quantile_over_time(0.99, data[100s]))"; - let plan = plan_without_raw_samples(&p99_workload("max", false), false); - assert!(!plan.stored_outputs.is_empty(), "panes built at ingestion"); + let plan = plan_without_raw_samples(&p99_workload("max", false), true); + assert!( + !plan.materialized_views().unwrap().is_empty(), + "tumbling windows kept at ingestion" + ); // Series 0..4 cross zero; series 4..8 are all negative. let (points, windows) = data_samples(8, 1_100, 100, |s, step| { let scale = (s + 1) as f64; @@ -329,7 +365,10 @@ async fn ddsketch_panes_over_signed_samples_answer_within_alpha() { }); let (app, _) = run(&plan, &points).await; let max = |p99s: &[f64]| p99s.iter().copied().fold(f64::NEG_INFINITY, f64::max); - assert_p99s_within_alpha(&answer(&app, query).await, &windows, max); + // Accuracy guarantees ignored: the selected sketch's own α bounds it. + let alpha = ddsketch_alpha(&plan); + assert!(alpha > 0.0, "a DDSketch plan"); + assert_p99s_within_alpha(&answer(&app, query).await, &windows, alpha, max); } /// A quantile of few samples interpolates between the two nearest, as @@ -340,7 +379,10 @@ async fn ddsketch_panes_over_signed_samples_answer_within_alpha() { async fn ddsketch_quantiles_interpolate_between_samples() { let query = "avg by (region, service, job) (quantile_over_time(0.5, data[100s]))"; let plan = plan_without_raw_samples(&quantile_workload("avg", 0.5, true), false); - assert!(!plan.stored_outputs.is_empty(), "panes built at ingestion"); + assert!( + !plan.materialized_views().unwrap().is_empty(), + "tumbling windows kept at ingestion" + ); // Eleven steps: the first, before the window, opens the panes (the // first pane after startup is never built). let (points, windows) = data_samples(8, 11, 10_000, |s, step| (s + 1) as f64 * step as f64); @@ -404,7 +446,7 @@ fn topk_workload(query: &str) -> PlanningWorkload { serde_json::from_str(&format!( r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ "query": {query:?}, - "demand": {{"fixed_interval": 1000}}, + "demand": {{"fixed_interval_at": {{"interval": 1000, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, @@ -509,7 +551,7 @@ async fn counter_topk_planned_without_guarantees_answers_exactly() { let workload: PlanningWorkload = serde_json::from_str(&format!( r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ "query": {query:?}, - "demand": {{"fixed_interval": 60000}}, + "demand": {{"fixed_interval_at": {{"interval": 60000, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3}}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100000.0}}}}, @@ -546,7 +588,7 @@ fn joint_workload(queries: &[&str], accuracy: &str) -> PlanningWorkload { .iter() .map(|query| { format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, "requirements": {accuracy}, "predictability": {{"predictable": {{"known_at": null}}}}}}"# ) @@ -556,7 +598,7 @@ fn joint_workload(queries: &[&str], accuracy: &str) -> PlanningWorkload { r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.6}}}}, "sources": {{"data": {{"series_count": 6, "sample_interval": 10000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, + "metric_type": "gauge", "column_signs": {{"value": "non_negative"}}}}}}}}}}"#, entries.join(",") )) .unwrap() @@ -607,10 +649,10 @@ fn grouped_series_samples() -> (Vec, Vec<(Labels, Vec)>) { } /// Quantile queries over one metric and window with different `by` labels -/// (ASAPQuery's seed-862 quantile queries), planned jointly, share one -/// per-series DDSketch, and each answers within 0.05 of its exact value. +/// (ASAPQuery's seed-862 quantile queries), planned jointly over per-series +/// DDSketch windows: each answers within 0.05 of its exact value. #[tokio::test] -async fn jointly_planned_quantiles_share_one_sketch_and_answer_within_alpha() { +async fn jointly_planned_quantiles_answer_within_alpha() { let queries = [ "quantile_over_time(0.9, data[100s])", "avg by (region) (quantile_over_time(0.99, data[100s]))", @@ -618,24 +660,6 @@ async fn jointly_planned_quantiles_share_one_sketch_and_answer_within_alpha() { let accuracy = r#"{"accuracy": {"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01, "accuracy_scope": "per_value"}}"#; let plan = plan_without_raw_samples(&joint_workload(&queries, accuracy), false); - let sketch_definitions: std::collections::BTreeSet<&str> = plan - .stored_outputs - .iter() - .filter(|output| { - let node = plan - .dag - .dag - .nodes - .iter() - .find(|n| n.id == output.node_id) - .unwrap(); - serde_json::to_string(&node.payload) - .unwrap() - .contains("DDSketch") - }) - .map(|output| output.coverage.definition_id.as_str()) - .collect(); - assert_eq!(sketch_definitions.len(), 1, "{sketch_definitions:?}"); let (points, windows) = grouped_series_samples(); let (app, _) = run(&plan, &points).await; @@ -703,13 +727,11 @@ async fn jointly_planned_fine_and_coarse_sums_answer_exactly() { } /// ASAPQuery's seed-862 top-k sums over one metric and two windows, -/// planned jointly: `[1m]` every second and `[20m]` every 30 s. The plan -/// stores per-series sums of one definition id in 1-second panes for the -/// first and 30-second panes for the second, with the same (no) label -/// constraints. Each query merges its own panes: every series' answer is -/// its sum over its query's whole window. +/// planned jointly: `[1m]` every second and `[20m]` every 30 s. Each +/// query merges its own tumbling windows: every series' answer is its sum +/// over its query's whole window. #[tokio::test] -async fn panes_of_one_definition_and_two_widths_are_stored_apart() { +async fn sums_over_two_window_widths_answer_exactly() { let queries = [ ( "topk by (region, service) (4, sum_over_time(data[1m]))", @@ -726,7 +748,7 @@ async fn panes_of_one_definition_and_two_widths_are_stored_apart() { .iter() .map(|(query, interval, _)| { format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": {interval}}}, + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": {interval}, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}"# @@ -737,25 +759,11 @@ async fn panes_of_one_definition_and_two_widths_are_stored_apart() { r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, "sources": {{"data": {{"series_count": 1000, "sample_interval": 100, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, + "metric_type": "gauge", "column_signs": {{"value": "non_negative"}}}}}}}}}}"#, entries.join(",") )) .unwrap(); let plan = plan_without_raw_samples(&workload, true); - // One definition id, two pane widths. - let mut widths: BTreeMap<&str, std::collections::BTreeSet> = BTreeMap::new(); - for output in &plan.stored_outputs { - for window in &output.inputs.raw_sample_windows { - widths - .entry(output.coverage.definition_id.as_str()) - .or_default() - .insert(window.width_ms); - } - } - assert!( - widths.values().any(|w| w.len() == 2), - "the case this tests: {widths:?}" - ); // Two series in one (region, service), ten samples per second each // over the last 21 minutes (the first minute, before the window, opens // the panes): 0.1 and 0.2, so 1 and 2 per second. @@ -784,3 +792,31 @@ async fn panes_of_one_definition_and_two_widths_are_stored_apart() { } } } + +/// Ingestion runs ahead of the queries it answers: after windows past the +/// evaluation time are refreshed (here 40 s more, so the plan's retention +/// alone, counted back from the newest window, would drop the oldest +/// windows a 100 s query at the earlier time reads), the query at the +/// earlier time still answers as before, within the default +/// `view_retention_margin_ms`. +#[tokio::test] +async fn a_query_behind_the_newest_window_is_still_answered() { + let query = "sum by (region) (sum_over_time(data[100s]))"; + let plan = plan_without_raw_samples(&joint_workload(&[query], "{}"), false); + let (points, _) = grouped_series_samples(); + let (app, state) = run(&plan, &points).await; + let before = answer(&app, query).await; + assert_eq!(before.len(), 2, "{before:?}"); + let later: Vec = points + .iter() + .filter(|p| p.timestamp_ms > EVALUATION_MS - 40_000) + .map(|p| Point { + labels: p.labels.clone(), + timestamp_ms: p.timestamp_ms + 40_000, + value: p.value, + }) + .collect(); + write_points(&app, &state, &later).await; + barrier(&app, EVALUATION_MS + 40_000).await; + assert_eq!(answer(&app, query).await, before); +} diff --git a/data_plane/tests/planner_executor_parity.rs b/data_plane/tests/planner_executor_parity.rs new file mode 100644 index 00000000..230f215f --- /dev/null +++ b/data_plane/tests/planner_executor_parity.rs @@ -0,0 +1,565 @@ +//! The data plane answers exactly what the Planner's executor answers for +//! the same installed plan and the same raw samples, on public PromQL +//! workloads: the #509 examples (`examples/`) and ASAPPlanner's mixed and +//! sketch-plan workloads (`tests/workloads/`, copied unchanged from +//! ASAPPlanner's `tools/dag-viewer/workloads/random-mixed-seed-862/` and +//! `crates/devtools/tests/sketch_plan_snapshots/workloads/`). +//! +//! Each workload is planned by the control plane twice, for a deployment +//! that keeps raw samples and for one that does not. The data plane +//! installs the plan with `--no-forward`, takes the samples by Remote +//! Write, closes the windows with the event-time barrier and answers each +//! query over HTTP. The reference runs the same plan in process as the +//! Planner's runtime tests do: every raw sample in an `InMemoryDataStore`, +//! each refresh group refreshed in one ingestion run per window, then one +//! query run. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::sync::Arc; + +use asap_executor::physical_planner::compile; +use asap_executor::planner::ir::operator::Source; +use asap_executor::planner::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; +use asap_executor::planner::ir::{NonASAPOp, Operator}; +use asap_executor::runtime::{Limits, RunContext, Scope}; +use asap_executor::sources::InMemoryDataStore; +use asap_executor::values::{Batch, Value}; +use asap_planner::{InstallablePlan, PlanningWorkload}; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use axum::Router; +use control_plane::{plan_workload, DeploymentInputs}; +use data_plane::engine::EngineOptions; +use data_plane::ingestion::IngestionOptions; +use data_plane::remote_write::{encode, Label, Sample, TimeSeries, WriteRequest}; +use data_plane::{router, DataPlane, EXECUTION_HEADER}; +use futures::{executor::block_on, StreamExt}; +use http_body_util::BodyExt; +use serde_json::Value as Json; +use tower::ServiceExt; + +/// The evaluation time: a whole hour in the past, on every window grid. +const EVALUATION_MS: i64 = 1_699_999_200_000; + +/// Series per metric. +const SERIES: usize = 6; + +type Labels = BTreeMap; +/// One answer row: its series labels and value. +type Answer = (Labels, f64); + +struct Point { + labels: Labels, + timestamp_ms: i64, + value: f64, +} + +fn workloads() -> Vec<(String, PlanningWorkload)> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files: Vec<_> = ["../examples", "tests/workloads"] + .iter() + .flat_map(|dir| std::fs::read_dir(root.join(dir)).unwrap()) + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension().is_some_and(|e| e == "json")) + .collect(); + files.sort(); + files + .into_iter() + .map(|path| { + let name = path.file_stem().unwrap().to_string_lossy().to_string(); + let text = std::fs::read_to_string(&path).unwrap(); + (name, serde_json::from_str(&text).unwrap()) + }) + .collect() +} + +/// Every metric the plan scans, with the union of its `Scan`s' columns. +fn scanned_schemas(plan: &InstallablePlan) -> BTreeMap { + let mut schemas: BTreeMap = BTreeMap::new(); + for node in &plan.dag.nodes { + let Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric }, + schema, + .. + }) = &node.payload + else { + continue; + }; + let known = schemas + .entry(metric.clone()) + .or_insert_with(|| schema.clone()); + for field in &schema.fields { + if !known.fields.iter().any(|f| f.name == field.name) { + known.fields.push(field.clone()); + } + } + } + schemas +} + +/// How far back from the evaluation the plan reads, at most: its views' +/// and scans' retention plus its widest refresh window. +fn reach_back_ms(plan: &InstallablePlan) -> i64 { + let retention = plan + .dag + .nodes + .iter() + .filter_map(|n| n.retention_ms) + .max() + .unwrap_or(0); + let widest = plan + .ingestion_refresh_groups() + .unwrap() + .iter() + .filter_map(|g| g.refresh_window.map(|w| w.width_ms)) + .max() + .unwrap_or(0); + (retention + 2 * widest).max(600_000) as i64 +} + +/// `SERIES` series of each scanned metric, every 10 s from `start_ms` +/// (exclusive) through the evaluation: a counter (`*_total`) grows, any +/// other metric takes pseudo-random values, some negative. +fn points(plan: &InstallablePlan, start_ms: i64) -> Vec { + let mut points = Vec::new(); + let mut state: u64 = 0x9e37_79b9_7f4a_7c15; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 11) as f64 / (1u64 << 53) as f64 + }; + for metric in scanned_schemas(plan).keys() { + let counter = metric.ends_with("_total"); + for series in 0..SERIES { + let labels: Labels = [ + ("__name__", metric.clone()), + ("job", format!("job{}", series % 3)), + ("instance", format!("i{series}")), + ("region", format!("r{}", series % 2)), + ("service", format!("s{}", series % 3)), + ] + .into_iter() + .map(|(n, v)| (n.to_string(), v)) + .collect(); + let mut total = 0.0; + let mut t = start_ms + 10_000; + while t <= EVALUATION_MS { + let value = if counter { + total += (next() * 10.0).floor(); + total + } else { + (next() * 1000.0).round() - 100.0 + }; + points.push(Point { + labels: labels.clone(), + timestamp_ms: t, + value, + }); + t += 10_000; + } + } + } + points.sort_by_key(|p| p.timestamp_ms); + points +} + +// ── The data plane ───────────────────────────────────────────────────── + +/// The data plane running `plan` with `--no-forward`, fed `points` by +/// Remote Write (one request per minute of event time) from `start_ms`, +/// with the barrier at the evaluation time. +async fn data_plane(plan: &InstallablePlan, points: &[Point], start_ms: i64) -> Router { + let options = EngineOptions { + no_forward: true, + ingestion: IngestionOptions { + allowed_lateness_ms: 0, + max_future_ms: 0, + stream_start_ms: Some(start_ms), + ..IngestionOptions::default() + }, + ..EngineOptions::default() + }; + let state = Arc::new( + DataPlane::with_options( + serde_json::to_string(plan).unwrap(), + "http://127.0.0.1:1".into(), + options, + ) + .unwrap(), + ); + let app = router(state.clone()); + let mut by_minute: BTreeMap>> = BTreeMap::new(); + for point in points { + by_minute + .entry(point.timestamp_ms.div_euclid(60_000)) + .or_default() + .entry(&point.labels) + .or_default() + .push(Sample { + timestamp: point.timestamp_ms, + value: point.value, + }); + } + for by_series in by_minute.into_values() { + let timeseries = by_series + .into_iter() + .map(|(labels, samples)| TimeSeries { + labels: labels + .iter() + .map(|(name, value)| Label { + name: name.clone(), + value: value.clone(), + }) + .collect(), + samples, + }) + .collect(); + let written = app + .clone() + .oneshot( + Request::post("/api/v1/write") + .body(Body::from(encode(&WriteRequest { timeseries }))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(written.status(), StatusCode::NO_CONTENT); + } + let barrier = app + .clone() + .oneshot( + Request::post(format!( + "/api/v1/precompute/watermark?time_ms={EVALUATION_MS}" + )) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(barrier.status(), StatusCode::OK); + app +} + +/// The data plane's answer to `query` at the evaluation time, or its error. +async fn served_answer(app: &Router, query: &str) -> Result, String> { + let uri = format!( + "/api/v1/query?{}", + form_urlencoded::Serializer::new(String::new()) + .append_pair("query", query) + .append_pair("time", &format!("{}", EVALUATION_MS as f64 / 1000.0)) + .finish() + ); + let response = app + .clone() + .oneshot(Request::get(uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + let status = response.status(); + let execution = response.headers().get(EXECUTION_HEADER).cloned(); + let body: Json = + serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap(); + if status != StatusCode::OK { + return Err(body["error"].as_str().unwrap_or_default().to_string()); + } + assert_eq!(execution.unwrap(), "plan"); + let mut answers: Vec = body["data"]["result"] + .as_array() + .unwrap() + .iter() + .map(|series| { + let labels: Labels = serde_json::from_value(series["metric"].clone()).unwrap(); + let value: f64 = match series["value"][1].as_str().unwrap() { + "NaN" => f64::NAN, + "+Inf" => f64::INFINITY, + "-Inf" => f64::NEG_INFINITY, + v => v.parse().unwrap(), + }; + (labels, value) + }) + .collect(); + answers.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(answers) +} + +// ── The reference: the Planner's executor in process ─────────────────── + +fn raw_tables(plan: &InstallablePlan, points: &[Point]) -> InMemoryDataStore { + let mut store = InMemoryDataStore::default(); + for (metric, schema) in scanned_schemas(plan) { + let schema = Arc::new(schema); + let rows = points + .iter() + .filter(|p| p.labels["__name__"] == metric) + .map(|point| { + schema + .fields + .iter() + .enumerate() + .map(|(index, field)| match field.name.as_str() { + PROMQL_SERIES_IDENTITY => { + Value::Utf8(serde_json::to_string(&point.labels).unwrap().into()) + } + _ if Some(index) == schema.time_index => { + Value::Timestamp(point.timestamp_ms) + } + "value" if field.dtype == FieldDataType::Plain(DataType::Float64) => { + Value::Float64(point.value) + } + label => { + Value::Utf8(point.labels.get(label).cloned().unwrap_or_default().into()) + } + }) + .collect() + }) + .collect(); + let batch = Batch::try_new(schema.clone(), rows).unwrap(); + store = store + .with_raw_data(Source::TimeSeries { metric }, schema, vec![batch]) + .unwrap(); + } + store +} + +/// Each query's answers (or the executor's error) from the plan run in +/// process over `points`: every refresh group's windows `(e - W, e]` with +/// `e ≡ p (mod W)`, `start_ms <= e - W` and `e <= T`, then one query run +/// at `T` per root. +fn reference_answers( + plan: &InstallablePlan, + points: &[Point], + start_ms: i64, +) -> Vec, String>> { + let dag = &plan.dag; + let views = plan.materialized_views().unwrap(); + let limits = Limits { max_bytes: 4 << 30 }; + let mut store = Arc::new(raw_tables(plan, points)); + for group in plan.ingestion_refresh_groups().unwrap() { + let window = group.refresh_window.unwrap(); + let (width, phase) = (window.width_ms as i64, window.phase_ms as i64); + let ids: Vec = group.views.iter().map(|v| v.node_id as u64).collect(); + let in_group: BTreeSet<_> = group.views.iter().map(|v| v.node_id).collect(); + let read: Vec<_> = views + .iter() + .filter(|v| { + !in_group.contains(&v.node_id) + && group + .views + .iter() + .any(|g| g.inputs.materialized_views.contains(&v.node_id)) + }) + .cloned() + .collect(); + let compiled = compile(dag, &read, &ids).unwrap(); + // The first window starting at or after `start_ms`. + let mut end = start_ms + width + (phase - start_ms - width).rem_euclid(width); + while end <= EVALUATION_MS { + let outputs = { + let instance = compiled.instantiate(store.clone()).unwrap(); + let context = RunContext::new( + Scope::Ingestion { + window_start_ms: end - width, + window_end_ms: end, + }, + limits.clone(), + ) + .unwrap(); + block_on( + instance + .execute_ingestion(compiled.roots(), context) + .unwrap(), + ) + .unwrap() + }; + let store = Arc::get_mut(&mut store).unwrap(); + for (root, batches) in compiled.roots().iter().zip(outputs) { + let view = group + .views + .iter() + .find(|v| v.node_id as u64 == *root) + .unwrap(); + let batches = batches.iter().map(|b| (**b).clone()).collect(); + store + .append_materialized_view(view, (end - width, end), batches) + .unwrap(); + } + end += width; + } + } + dag.roots + .iter() + .map(|&root| { + let read: Vec<_> = views + .iter() + .filter(|v| reaches_without_passing_a_view(plan, root, v.node_id, &views)) + .cloned() + .collect(); + let compiled = compile(dag, &read, &[root as u64]).map_err(|e| e.to_string())?; + let instance = compiled + .instantiate(store.clone()) + .map_err(|e| e.to_string())?; + let context = RunContext::new( + Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }, + limits.clone(), + ) + .unwrap(); + let schema = compiled.output_schema(root as u64).unwrap(); + let batches = block_on(async { + let mut output = instance + .execute(compiled.roots(), context) + .map_err(|e| e.to_string())? + .remove(0); + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + batches.push((*batch.map_err(|e| e.to_string())?).clone()); + } + Ok::<_, String>(batches) + })?; + Ok(answers(&schema, &batches)) + }) + .collect() +} + +/// Whether `view` is a view slot of `root`'s query run: reached from it +/// without passing another view. +fn reaches_without_passing_a_view( + plan: &InstallablePlan, + root: usize, + view: usize, + views: &[asap_planner::MaterializedView], +) -> bool { + let is_view = |id: usize| views.iter().any(|v| v.node_id == id); + let mut pending = vec![root]; + let mut seen = BTreeSet::new(); + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + if id == view { + return true; + } + if is_view(id) { + continue; + } + pending.extend( + plan.dag + .edges + .iter() + .filter(|e| e.consumer == id) + .map(|e| e.producer), + ); + } + false +} + +/// Answer rows: the series identity's labels and other non-empty text +/// columns, the Float64 value; sorted by labels. +fn answers(schema: &Schema, batches: &[Batch]) -> Vec { + let mut answers: Vec = batches + .iter() + .flat_map(Batch::to_rows) + .map(|row| { + let mut labels = Labels::new(); + let mut value = f64::NAN; + for (field, cell) in schema.fields.iter().zip(&row) { + match cell { + Value::Utf8(text) if field.name == PROMQL_SERIES_IDENTITY => { + let identity: Labels = serde_json::from_str(text).unwrap(); + labels.extend(identity); + } + Value::Utf8(text) if !text.is_empty() => { + labels.insert(field.name.clone(), text.to_string()); + } + Value::Float64(v) => value = *v, + _ => {} + } + } + (labels, value) + }) + .collect(); + answers.sort_by(|a, b| a.0.cmp(&b.0)); + answers +} + +fn same(a: &[Answer], b: &[Answer]) -> bool { + a.len() == b.len() + && a.iter().zip(b).all(|(x, y)| { + x.0 == y.0 + && (x.1 == y.1 + || (x.1.is_nan() && y.1.is_nan()) + || (x.1 - y.1).abs() <= 1e-9 * y.1.abs().max(1.0)) + }) +} + +/// Every public workload, planned with and without raw samples kept: each +/// query the reference answers, the data plane answers the same; each +/// query the reference cannot answer, the data plane does not answer +/// either. At least ten workload plans are compared query by query. +#[tokio::test(flavor = "multi_thread")] +async fn the_data_plane_answers_what_the_planner_executor_answers() { + let mut compared_plans = 0; + let mut compared_queries = 0; + let mut compared_non_empty = 0; + let mut report = Vec::new(); + let mut mismatches = Vec::new(); + for (name, workload) in workloads() { + for raw_data_retained in [true, false] { + let label = format!("{name} (raw data retained: {raw_data_retained})"); + let mut inputs = DeploymentInputs::default(); + inputs.deployment_capabilities.raw_data_retained = raw_data_retained; + let plan = match plan_workload(&workload, &inputs) { + Ok((plan, _)) => plan, + Err(error) => { + report.push(format!("{label}: not planned: {error}")); + continue; + } + }; + let start_ms = EVALUATION_MS - reach_back_ms(&plan); + let points = points(&plan, start_ms); + let expected = reference_answers(&plan, &points, start_ms); + let app = data_plane(&plan, &points, start_ms).await; + let mut answered = 0; + let mut non_empty = 0; + for (query, expected) in plan.query_texts.iter().zip(&expected) { + let served = served_answer(&app, query).await; + match (expected, &served) { + (Ok(expected), Ok(served)) if same(served, expected) => { + answered += 1; + non_empty += usize::from(!served.is_empty()); + } + (Err(_), Err(_)) => { + report.push(format!("{label}: {query}: neither answers: {expected:?}")) + } + _ => mismatches.push(format!( + "{label}: {query}:\n data plane {served:?}\n reference {expected:?}" + )), + } + } + compared_queries += answered; + compared_non_empty += non_empty; + if answered > 0 { + compared_plans += 1; + } + report.push(format!( + "{label}: {answered}/{} queries equal ({non_empty} with rows), {} nodes, {} views", + plan.query_texts.len(), + plan.dag.nodes.len(), + plan.materialized_views().unwrap().len() + )); + } + } + println!("{}", report.join("\n")); + assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); + // Equal answers must not be equal only because both are empty. + assert!( + compared_non_empty * 10 >= compared_queries * 9, + "only {compared_non_empty} of {compared_queries} equal answers have rows" + ); + assert!( + compared_plans >= 10, + "only {compared_plans} plans compared ({compared_queries} queries)" + ); +} diff --git a/data_plane/tests/precompute.rs b/data_plane/tests/precompute.rs deleted file mode 100644 index 19a8c7c9..00000000 --- a/data_plane/tests/precompute.rs +++ /dev/null @@ -1,535 +0,0 @@ -mod common; - -use std::collections::BTreeMap; - -use asap_executor::planner::workload::{PlanningWorkload, Query}; -use asap_planner::InstallablePlan; -use control_plane::plan_workload; -use data_plane::precompute::{pane_end, Precompute, PrecomputeOptions, RawSample}; -use data_plane::sds::Sds; - -const MINUTE: i64 = 60_000; -const PANE: i64 = 10 * MINUTE; - -const DAY: i64 = 24 * 60 * MINUTE; -/// The Prometheus stale marker's bits. -const STALE_MARKER: u64 = 0x7ff0_0000_0000_0002; - -fn precompute(plan: InstallablePlan) -> (Precompute, Sds) { - precompute_with(plan, 0) -} - -fn precompute_with(plan: InstallablePlan, allowed_lateness_ms: u64) -> (Precompute, Sds) { - let sds = Sds::from_plan(&plan).unwrap(); - let dag = plan.dag.dag; - let options = PrecomputeOptions { - allowed_lateness_ms, - ..PrecomputeOptions::default() - }; - (Precompute::new(dag, &sds, options).unwrap(), sds) -} - -fn sample(metric: &str, job: &str, timestamp_ms: i64, value: f64) -> RawSample { - RawSample { - labels: data_plane::series_ids::intern(&BTreeMap::from([ - ("__name__".to_string(), metric.to_string()), - ("job".to_string(), job.to_string()), - ])), - timestamp_ms, - value, - } -} - -/// One sample every 15 s for each job, at `from`, `from + 15 s`, ... before `to`. -fn samples(metric: &str, jobs: &[&str], from: i64, to: i64) -> Vec { - (from..to) - .step_by(15_000) - .flat_map(|t| jobs.iter().map(move |job| sample(metric, job, t, t as f64))) - .collect() -} - -/// Panes are built once the watermark passes them, one state per series. -/// The first pane seen may be partial, so it is skipped. -#[test] -fn builds_panes_after_the_first_as_the_watermark_passes() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - // The first sample, at 1 min, opens pane (0, 10 min], which is skipped. - engine.ingest(&samples("latency_ms", &["a", "b"], MINUTE, 35 * MINUTE)); - assert_eq!( - engine.store.get_pane(&id, PANE).map(|_| ()), - None, - "first pane" - ); - for end in [2 * PANE, 3 * PANE] { - let pane = engine.store.get_pane(&id, end).expect("built pane"); - assert_eq!(pane.num_rows(), 2, "one state per series"); - } - assert_eq!( - engine.store.get_pane(&id, 4 * PANE).map(|_| ()), - None, - "open pane" - ); - let stats = engine.stats; - assert_eq!(stats.panes_built, 2); - assert_eq!(stats.panes_failed, 0); - // Samples in (0, 10 min] are dropped with the first pane. - assert_eq!(stats.samples_skipped_first_pane, 2 * 37); - assert_eq!(stats.samples_dropped_late, 0); - assert_eq!(stats.samples_accepted, 2 * 99); -} - -/// Samples of other metrics are ignored; samples for a built pane are dropped. -#[test] -fn ignores_other_metrics_and_drops_late_samples() { - let (mut engine, _) = precompute(common::plan("example4b", false)); - engine.ingest(&samples("latency_ms", &["a"], 0, 25 * MINUTE)); - let before = engine.stats; - engine.ingest(&[sample("other", "a", 25 * MINUTE, 1.0)]); - assert_eq!(engine.stats.samples_unrouted, before.samples_unrouted + 1); - assert_eq!(engine.stats.samples_accepted, before.samples_accepted); - engine.ingest(&[sample("latency_ms", "a", 15 * MINUTE, 1.0)]); - assert_eq!( - engine.stats.samples_dropped_late, - before.samples_dropped_late + 1 - ); - assert_eq!( - engine.stats.samples_skipped_first_pane, - before.samples_skipped_first_pane - ); -} - -/// A pane without samples is stored empty, so a query can tell "no data" -/// from "not built". -#[test] -fn stores_empty_panes() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest(&samples("latency_ms", &["a"], 0, 20 * MINUTE)); - engine.ingest(&[sample("latency_ms", "a", 45 * MINUTE, 1.0)]); - assert_eq!(engine.store.get_pane(&id, 2 * PANE).unwrap().num_rows(), 1); - assert_eq!(engine.store.get_pane(&id, 3 * PANE).unwrap().num_rows(), 0); - assert_eq!(engine.store.get_pane(&id, 4 * PANE).unwrap().num_rows(), 0); -} - -/// The plan's label matchers select which series a pane summarizes. -#[test] -fn applies_label_matchers() { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../examples/planner-layering-example4b.json"); - let mut workload: PlanningWorkload = - serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("quantile_over_time(0.99, latency_ms{job=\"a\"}[60m])".into()); - let (mut engine, sds) = precompute(plan_workload(&workload, &common::inputs(false)).unwrap()); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest(&samples("latency_ms", &["a", "b"], 0, 25 * MINUTE)); - assert_eq!(engine.store.get_pane(&id, 2 * PANE).unwrap().num_rows(), 1); -} - -/// Panes are `(end - width, end]`, like PromQL ranges: a sample at a -/// boundary belongs to the pane it ends. -#[test] -fn panes_are_left_open() { - assert_eq!(pane_end(1, PANE), Some(PANE)); - assert_eq!(pane_end(PANE, PANE), Some(PANE)); - assert_eq!(pane_end(PANE + 1, PANE), Some(2 * PANE)); - assert_eq!(pane_end(0, PANE), Some(0)); - assert_eq!(pane_end(-1, PANE), Some(0)); - assert_eq!(pane_end(-PANE, PANE), Some(-PANE)); -} - -/// The engine puts a sample at a pane boundary into the pane it ends, and a -/// sample one millisecond later into the next pane. -#[test] -fn engine_assigns_boundary_samples_to_the_pane_they_end() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest(&[ - // Opens the skipped first pane (0, 10 min]. - sample("latency_ms", "a", MINUTE, 1.0), - sample("latency_ms", "a", 15 * MINUTE, 1.0), - sample("latency_ms", "b", 2 * PANE, 1.0), - sample("latency_ms", "c", 2 * PANE + 1, 1.0), - // Moves the watermark past (20 min, 30 min]. - sample("latency_ms", "a", 45 * MINUTE, 1.0), - ]); - // (10 min, 20 min]: series a and b, the one at its end included. - assert_eq!(engine.store.get_pane(&id, 2 * PANE).unwrap().num_rows(), 2); - // (20 min, 30 min]: only c, one millisecond after the boundary. - assert_eq!(engine.store.get_pane(&id, 3 * PANE).unwrap().num_rows(), 1); -} - -/// Samples of a metric no definition reads do not move the watermark, even -/// far in the future: the next real sample is still accepted. -#[test] -fn unrouted_future_sample_does_not_close_panes() { - let (mut engine, _) = precompute(common::plan("example4b", false)); - let now = 30 * MINUTE; - engine.ingest_at(&samples("latency_ms", &["a"], 0, 25 * MINUTE), now); - let built = engine.stats.panes_built; - engine.ingest_at(&[sample("other_metric", "a", 30 * DAY, 1.0)], 30 * DAY); - assert_eq!(engine.stats.panes_built, built, "no pane closed"); - assert_eq!(engine.stats.samples_unrouted, 1); - engine.ingest_at(&[sample("latency_ms", "a", 26 * MINUTE, 1.0)], now); - assert_eq!(engine.stats.samples_dropped_late, 0); -} - -/// A routed sample further ahead of the wall clock than `max_future_ms` is -/// dropped and counted, and does not move the watermark. -#[test] -fn drops_samples_beyond_the_future_bound() { - let (mut engine, _) = precompute(common::plan("example4b", false)); - let now = 30 * MINUTE; - let bound = PrecomputeOptions::default().max_future_ms as i64; - engine.ingest_at(&samples("latency_ms", &["a"], 0, 25 * MINUTE), now); - let built = engine.stats.panes_built; - engine.ingest_at(&[sample("latency_ms", "a", now + bound + 1, 1.0)], now); - assert_eq!(engine.stats.samples_dropped_future, 1); - assert_eq!(engine.stats.panes_built, built); - // Exactly at the bound is accepted. - engine.ingest_at(&[sample("latency_ms", "a", now + bound, 1.0)], now); - assert_eq!(engine.stats.samples_dropped_future, 1); -} - -/// When the watermark jumps by 30 days, only the panes still retained are -/// built; the rest are skipped and counted, not built and evicted. -#[test] -fn watermark_jump_builds_only_retained_panes() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest_at(&samples("latency_ms", &["a"], 0, 25 * MINUTE), 30 * DAY); - let built = engine.stats.panes_built; - engine.ingest_at(&[sample("latency_ms", "a", 30 * DAY, 1.0)], 30 * DAY); - // The plan keeps Example 4b's panes 60 min back, plus one pane: the - // seven panes ending at 30 days - 60 min through 30 days. - assert_eq!(engine.stats.panes_built - built, 7); - assert_eq!( - engine.stats.panes_skipped_expired, - (30 * DAY - 90 * MINUTE) as u64 / PANE as u64 - ); - assert!(engine.store.get_pane(&id, 30 * DAY).is_some()); - assert!(engine.store.get_pane(&id, 30 * DAY - 7 * PANE).is_none()); - assert!(engine.store.get_pane(&id, 2 * PANE).is_none()); - assert_eq!(engine.store.len(), 7); -} - -/// Timestamps at the ends of `i64` neither overflow nor loop. -#[test] -fn extreme_timestamps_do_not_overflow() { - assert_eq!(pane_end(i64::MIN, PANE), None); - assert_eq!(pane_end(i64::MAX, PANE), None); - let (mut engine, _) = precompute(common::plan("example4b", false)); - engine.ingest_at( - &[ - sample("latency_ms", "a", i64::MIN, 1.0), - sample("latency_ms", "a", i64::MIN + 1, 1.0), - sample("latency_ms", "a", 0, 1.0), - sample("latency_ms", "a", i64::MAX - 1, 1.0), - sample("latency_ms", "a", i64::MAX, 1.0), - ], - i64::MAX, - ); - assert!(engine.stats.panes_built <= 7); -} - -/// A pane of zero width is refused when the engine is built. -#[test] -fn rejects_zero_width_panes() { - let plan = common::plan("example4b", false); - let mut sds = Sds::from_plan(&plan).unwrap(); - let dag = plan.dag.dag; - for definition in sds.definitions.values_mut() { - definition.pane_ms = 0; - } - assert!(Precompute::new(dag, &sds, PrecomputeOptions::default()).is_err()); -} - -/// Panes older than the plan's retention are evicted as the watermark -/// advances. -#[test] -fn evicts_panes_past_retention() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest(&samples("latency_ms", &["a"], 0, 25 * MINUTE)); - assert!(engine.store.get_pane(&id, 2 * PANE).is_some()); - // Watermark 100 min, retention 60 min plus one pane: the oldest pane - // read by the evaluation at 90 min is (30, 40 min], so panes ending - // before 40 min are evicted. - engine.ingest(&[sample("latency_ms", "a", 100 * MINUTE, 1.0)]); - assert!(engine.store.get_pane(&id, 2 * PANE).is_none()); - assert!(engine.store.get_pane(&id, 3 * PANE).is_none()); - assert!(engine.store.get_pane(&id, 4 * PANE).is_some()); - assert!(engine.store.get_pane(&id, 10 * PANE).is_some()); - assert_eq!(engine.store.len(), 7); -} - -/// Out-of-order samples within the allowed lateness reach their pane. -#[test] -fn accepts_out_of_order_samples_within_lateness() { - let (mut engine, sds) = precompute_with(common::plan("example4b", false), 5 * MINUTE as u64); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest(&[sample("latency_ms", "a", MINUTE, 1.0)]); - engine.ingest(&[sample("latency_ms", "a", 24 * MINUTE, 1.0)]); - // Watermark 19 min: (10, 20 min] is still open. - engine.ingest(&[sample("latency_ms", "b", 15 * MINUTE, 1.0)]); - engine.ingest(&[sample("latency_ms", "a", 26 * MINUTE, 1.0)]); - // Watermark 21 min: (10, 20 min] holds b's late sample. - assert_eq!(engine.store.get_pane(&id, 2 * PANE).unwrap().num_rows(), 1); - assert_eq!(engine.stats.samples_dropped_late, 0); - engine.ingest(&[sample("latency_ms", "c", 18 * MINUTE, 1.0)]); - assert_eq!(engine.stats.samples_dropped_late, 1); -} - -/// Stale markers are not samples: a series whose only sample in a pane is -/// a stale marker has no state, and one with a value and a marker keeps it. -#[test] -fn stale_markers_are_not_samples() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - let stale = f64::from_bits(STALE_MARKER); - engine.ingest(&[ - sample("latency_ms", "a", MINUTE, 1.0), - sample("latency_ms", "a", 12 * MINUTE, 1.0), - sample("latency_ms", "a", 13 * MINUTE, stale), - sample("latency_ms", "b", 14 * MINUTE, stale), - sample("latency_ms", "a", 25 * MINUTE, 1.0), - ]); - assert_eq!(engine.store.get_pane(&id, 2 * PANE).unwrap().num_rows(), 1); -} - -/// One series stamped ahead of the wall clock, though within -/// `max_future_ms`, does not close panes early: the watermark is bounded by -/// the wall clock, so an on-time sample of the open pane is still accepted. -#[test] -fn series_ahead_of_the_clock_does_not_close_panes_early() { - let (mut engine, sds) = precompute_with(common::plan("example4b", false), 15_000); - let id = sds.definitions.keys().next().unwrap().clone(); - let base = PANE; - // Opens the skipped first pane (0, 10 min]. - engine.ingest_at(&[sample("latency_ms", "a", MINUTE, 1.0)], MINUTE); - let now = base + 9 * MINUTE; - engine.ingest_at(&[sample("latency_ms", "b", base + 13 * MINUTE, 1.0)], now); - assert!( - engine.store.get_pane(&id, base + PANE).is_none(), - "(10, 20 min] still open" - ); - engine.ingest_at(&[sample("latency_ms", "a", base + 9 * MINUTE, 1.0)], now); - assert_eq!(engine.stats.samples_dropped_late, 0); - assert_eq!(engine.stats.samples_accepted, 2); -} - -/// A far-future sample of a metric no definition reads is unrouted, not -/// dropped as future. -#[test] -fn far_future_unrouted_sample_counts_as_unrouted() { - let (mut engine, _) = precompute(common::plan("example4b", false)); - engine.ingest_at(&[sample("other_metric", "a", 30 * DAY, 1.0)], 0); - assert_eq!(engine.stats.samples_unrouted, 1); - assert_eq!(engine.stats.samples_dropped_future, 0); -} - -/// Summaries built in a query run are not built at ingestion. -#[test] -fn does_not_build_query_run_outputs() { - let mut plan = common::plan("example4b", false); - for output in &mut plan.stored_outputs { - output.built_at = asap_planner::StoredOutputBuild::QueryRun; - } - let (mut engine, _) = precompute(plan); - engine.ingest(&samples("latency_ms", &["a"], 0, 45 * MINUTE)); - assert_eq!(engine.stats.panes_built, 0); - assert_eq!(engine.stats.samples_query_run_only, 180); -} - -/// Retention is measured from the newest evaluation the stored panes can -/// serve, the end of the newest closed pane, not from the watermark: with -/// the watermark at 74:45 the newest closed pane ends at 70 min, and an -/// evaluation at 70 min still reads the pane ending at 20 min. -#[test] -fn keeps_every_pane_the_newest_served_evaluation_reads() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest(&samples("latency_ms", &["a"], MINUTE, 75 * MINUTE)); - for end in (2..=7).map(|i| i * PANE) { - assert!( - engine.store.get_pane(&id, end).is_some(), - "pane ending at {end}" - ); - } - assert_eq!(engine.stats.panes_skipped_expired, 0); -} - -/// An evaluation one pane behind the newest closed pane still has every -/// pane it reads: with the watermark at 84:45 the newest closed pane ends -/// at 80 min, and the evaluation at 70 min reads the panes ending 20..70. -/// The extra pane is the only margin: the store stays bounded. -#[test] -fn keeps_one_pane_of_margin_beyond_the_retention() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest(&samples("latency_ms", &["a"], MINUTE, 85 * MINUTE)); - for end in (2..=8).map(|i| i * PANE) { - assert!( - engine.store.get_pane(&id, end).is_some(), - "pane ending at {end}" - ); - } - engine.ingest(&samples("latency_ms", &["a"], 85 * MINUTE, 300 * MINUTE)); - // Panes ending 230..290 min: the six the evaluation at 290 reads, plus - // the one more the evaluation at 280 reads. - assert_eq!(engine.store.len(), 7); -} - -/// Samples of a metric only stored outputs built in a `QueryRun` read (B3) are not buffered -/// but move the watermark, which says when those summaries are closed. -#[test] -fn query_run_metrics_move_the_watermark() { - let (mut engine, _) = precompute_with(common::plan("example4-b3", true), 0); - assert_eq!(engine.watermark_ms(), None); - engine.ingest(&samples("latency_ms", &["a"], 0, 3 * MINUTE)); - assert_eq!(engine.stats.samples_query_run_only, 12); - assert_eq!(engine.stats.samples_accepted, 0); - assert_eq!(engine.watermark_ms(), Some(3 * MINUTE - 15_000)); -} - -/// Samples of a metric only `QueryRun` stored outputs read get the same future bound and -/// wall-clock cap as routed samples: one far ahead is dropped, and one -/// ahead of the clock but within the bound moves the watermark only to the -/// clock. -#[test] -fn query_run_metrics_are_bounded_like_routed_samples() { - let (mut engine, _) = precompute_with(common::plan("example4-b3", true), 0); - let now = 10 * MINUTE; - engine.ingest_at(&[sample("latency_ms", "a", 30 * DAY, 1.0)], now); - assert_eq!(engine.stats.samples_dropped_future, 1); - assert_eq!(engine.watermark_ms(), None); - engine.ingest_at(&[sample("latency_ms", "a", now + 2 * MINUTE, 1.0)], now); - assert_eq!(engine.stats.samples_query_run_only, 1); - assert_eq!(engine.watermark_ms(), Some(now)); -} - -/// An event-time barrier closes the last pane of a series that stopped -/// sending, without a later sample. -#[test] -fn barrier_closes_the_last_pane_of_a_stopped_series() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest_at( - &samples("latency_ms", &["a"], MINUTE, 25 * MINUTE), - 31 * MINUTE, - ); - assert!(engine.store.get_pane(&id, 2 * PANE).is_some()); - assert!( - engine.store.get_pane(&id, 3 * PANE).is_none(), - "(20, 30 min] open" - ); - assert_eq!(engine.advance_watermark_at(3 * PANE, 31 * MINUTE), 3 * PANE); - assert_eq!(engine.store.get_pane(&id, 3 * PANE).unwrap().num_rows(), 1); -} - -/// A repeated or lower barrier does nothing, and a barrier ahead of the -/// wall clock is capped at it. -#[test] -fn barrier_is_monotonic_and_capped_at_the_wall_clock() { - let (mut engine, sds) = precompute(common::plan("example4b", false)); - let id = sds.definitions.keys().next().unwrap().clone(); - engine.ingest_at( - &samples("latency_ms", &["a"], MINUTE, 25 * MINUTE), - 31 * MINUTE, - ); - assert_eq!( - engine.advance_watermark_at(4 * PANE, 31 * MINUTE), - 31 * MINUTE - ); - assert!(engine.store.get_pane(&id, 3 * PANE).is_some()); - assert!( - engine.store.get_pane(&id, 4 * PANE).is_none(), - "capped at 31 min" - ); - let stats = engine.stats; - assert_eq!( - engine.advance_watermark_at(31 * MINUTE, 31 * MINUTE), - 31 * MINUTE - ); - assert_eq!( - engine.advance_watermark_at(2 * PANE, 31 * MINUTE), - 31 * MINUTE - ); - assert_eq!(engine.stats, stats); -} - -/// A barrier at `i64::MAX` terminates and builds only retained panes. -#[test] -fn barrier_at_i64_max_terminates() { - let (mut engine, _) = precompute(common::plan("example4b", false)); - engine.ingest_at( - &samples("latency_ms", &["a"], MINUTE, 25 * MINUTE), - i64::MAX, - ); - let built = engine.stats.panes_built; - assert_eq!(engine.advance_watermark_at(i64::MAX, i64::MAX), i64::MAX); - assert!(engine.stats.panes_built - built <= 7); -} - -/// Input stays open after a barrier: a later sample is accepted, and one at -/// or before the barrier counts as late. -#[test] -fn input_stays_open_after_a_barrier() { - let (mut engine, _) = precompute(common::plan("example4b", false)); - engine.ingest_at( - &samples("latency_ms", &["a"], MINUTE, 25 * MINUTE), - 31 * MINUTE, - ); - engine.advance_watermark_at(3 * PANE, 31 * MINUTE); - let accepted = engine.stats.samples_accepted; - engine.ingest_at(&[sample("latency_ms", "a", 35 * MINUTE, 1.0)], 36 * MINUTE); - assert_eq!(engine.stats.samples_accepted, accepted + 1); - engine.ingest_at(&[sample("latency_ms", "a", 29 * MINUTE, 1.0)], 36 * MINUTE); - assert_eq!(engine.stats.samples_dropped_late, 1); -} - -/// A producer that declares where its stream starts (`stream_start_ms`): -/// every sample at or after it reaches the data plane. The first pane seen -/// is then complete when it starts at or after the declared start, and is -/// built; one that starts before it is still skipped. -#[test] -fn the_first_pane_is_built_when_it_starts_at_the_declared_stream_start() { - let plan = common::plan("example4b", false); - let sds = Sds::from_plan(&plan).unwrap(); - let id = sds.definitions.keys().next().unwrap().clone(); - let with_start = |stream_start_ms| { - let options = PrecomputeOptions { - allowed_lateness_ms: 0, - stream_start_ms: Some(stream_start_ms), - ..PrecomputeOptions::default() - }; - Precompute::new(plan.dag.dag.clone(), &sds, options).unwrap() - }; - // The stream starts at the pane boundary 10 min: pane (10, 20 min] is - // complete. - let mut engine = with_start(PANE); - engine.ingest(&samples( - "latency_ms", - &["a", "b"], - PANE + 15_000, - 35 * MINUTE, - )); - let pane = engine - .store - .get_pane(&id, 2 * PANE) - .expect("first pane built"); - assert_eq!(pane.num_rows(), 2); - assert_eq!(engine.stats.samples_skipped_first_pane, 0); - // The stream starts at 15 min, inside (10, 20 min]: that pane may have - // missed samples before 15 min, so it is skipped. - let mut engine = with_start(PANE + 5 * MINUTE); - engine.ingest(&samples( - "latency_ms", - &["a", "b"], - PANE + 5 * MINUTE, - 35 * MINUTE, - )); - assert!(engine.store.get_pane(&id, 2 * PANE).is_none()); - assert!(engine.store.get_pane(&id, 3 * PANE).is_some()); -} diff --git a/data_plane/tests/query.rs b/data_plane/tests/query.rs deleted file mode 100644 index 2fc3b87f..00000000 --- a/data_plane/tests/query.rs +++ /dev/null @@ -1,351 +0,0 @@ -mod common; - -use std::collections::BTreeMap; - -use asap_planner::InstallablePlan; -use data_plane::precompute::{Precompute, PrecomputeOptions, RawSample}; -use data_plane::query::{ - BoundRawSampleWindow, LabelMatcher, MatchOp, QueryEngine, ResultSample, Unanswered, -}; -use data_plane::sds::Sds; - -const MINUTE: i64 = 60_000; - -struct Fixture { - precompute: Precompute, - query: QueryEngine, -} - -fn fixture(plan: InstallablePlan) -> Fixture { - let sds = Sds::from_plan(&plan).unwrap(); - let options = PrecomputeOptions { - allowed_lateness_ms: 0, - ..PrecomputeOptions::default() - }; - Fixture { - precompute: Precompute::new(plan.dag.dag.clone(), &sds, options).unwrap(), - query: QueryEngine::new(&plan, &sds), - } -} - -/// Samples every 15 s in `[from, to)` for each `(metric, labels)`, with -/// value `f(series index, t)`. -fn samples( - series: &[(&str, &[(&str, &str)])], - from: i64, - to: i64, - f: impl Fn(usize, i64) -> f64, -) -> Vec { - let series: Vec<_> = series - .iter() - .map(|(metric, labels)| { - let mut map = BTreeMap::from([("__name__".to_string(), metric.to_string())]); - map.extend(labels.iter().map(|(k, v)| (k.to_string(), v.to_string()))); - data_plane::series_ids::intern(&map) - }) - .collect(); - (from..to) - .step_by(15_000) - .flat_map(|t| { - series - .iter() - .enumerate() - .map(move |(i, labels)| (i, labels.clone(), t)) - }) - .map(|(i, labels, timestamp_ms)| RawSample { - labels, - timestamp_ms, - value: f(i, timestamp_ms), - }) - .collect() -} - -/// What a deployment fetches for each bound raw-sample window: its metric's samples in -/// `(start, end]`. -fn fetch(ranges: &[BoundRawSampleWindow], all: &[RawSample]) -> BTreeMap> { - ranges - .iter() - .map(|range| { - let samples = all - .iter() - .filter(|s| { - s.labels["__name__"] == range.metric - && range.start_ms < s.timestamp_ms - && s.timestamp_ms <= range.end_ms - }) - .cloned() - .collect(); - (range.node, samples) - }) - .collect() -} - -fn answer(f: &mut Fixture, query: &str, time_ms: i64, all: &[RawSample]) -> Vec { - let prepared = f - .query - .bound_raw_sample_windows(query, time_ms, &f.precompute.store, f.precompute.progress()) - .unwrap(); - let raw = fetch(&prepared.windows, all); - let mut result = f - .query - .evaluate( - prepared.root, - time_ms, - &mut f.precompute.store, - &prepared.progress, - &raw, - ) - .unwrap(); - result.sort_by(|a, b| a.labels.cmp(&b.labels)); - result -} - -const P99: &str = "quantile_over_time(0.99, latency_ms[60m])"; - -/// Example 4b's p99 merged from six stored 10-minute KLL panes equals one -/// query-time build over the same hour of raw samples (KLL is exact below k). -#[test] -fn stored_panes_answer_like_a_single_build() { - let series: &[(&str, &[(&str, &str)])] = &[ - ("latency_ms", &[("job", "a")]), - ("latency_ms", &[("job", "b")]), - ]; - // Values cycle through 0..240 per series, offset by 1000 for series b. - let all = samples(series, MINUTE, 75 * MINUTE, |i, t| { - (i * 1000) as f64 + ((t / 15_000) % 240) as f64 - }); - let mut panes = fixture(common::plan("example4b", false)); - assert!( - panes - .query - .bound_raw_sample_windows( - P99, - 70 * MINUTE, - &panes.precompute.store, - Default::default() - ) - .is_err(), - "no panes before ingestion" - ); - panes.precompute.ingest(&all); - let ranges = panes - .query - .bound_raw_sample_windows( - P99, - 70 * MINUTE, - &panes.precompute.store, - Default::default(), - ) - .unwrap() - .windows; - assert!(ranges.is_empty(), "every input is a stored pane"); - let from_panes = answer(&mut panes, P99, 70 * MINUTE, &all); - - let mut single = fixture(common::plan("example4b", true)); - let ranges = single - .query - .bound_raw_sample_windows( - P99, - 70 * MINUTE, - &single.precompute.store, - Default::default(), - ) - .unwrap() - .windows; - assert_eq!( - ranges - .iter() - .map(|r| (r.start_ms, r.end_ms)) - .collect::>(), - vec![(10 * MINUTE, 70 * MINUTE)], - "one raw hour, left-open" - ); - let from_raw = answer(&mut single, P99, 70 * MINUTE, &all); - - assert_eq!(from_panes, from_raw); - assert_eq!(from_panes.len(), 2); - for (sample, base) in from_panes.iter().zip([0.0, 1000.0]) { - assert!( - sample.value >= base + 236.0 && sample.value <= base + 239.0, - "{sample:?}" - ); - } -} - -/// Example 1 runs entirely at query time over raw samples: each counter grows -/// by 1 per second, so `sum by (job) (rate(...[1m]))` is the series count. -#[test] -fn query_time_plans_read_raw_ranges() { - let series: &[(&str, &[(&str, &str)])] = &[ - ("http_requests_total", &[("job", "a"), ("instance", "1")]), - ("http_requests_total", &[("job", "a"), ("instance", "2")]), - ("http_requests_total", &[("job", "b"), ("instance", "1")]), - ]; - let all = samples(series, 0, 10 * MINUTE, |_, t| t as f64 / 1000.0); - let mut f = fixture(common::plan("example1", true)); - let result = answer( - &mut f, - "sum by (job) (rate(http_requests_total[1m]))", - 5 * MINUTE, - &all, - ); - let by_job: Vec<(String, f64)> = result - .iter() - .map(|s| (s.labels["job"].clone(), s.value)) - .collect(); - assert_eq!(by_job.len(), 2); - assert_eq!(by_job[0].0, "a"); - assert!((by_job[0].1 - 2.0).abs() < 1e-9, "{by_job:?}"); - assert_eq!(by_job[1].0, "b"); - assert!((by_job[1].1 - 1.0).abs() < 1e-9, "{by_job:?}"); - - let topk = answer( - &mut f, - "topk by (job) (10, sum_over_time(http_requests_total[1m]))", - 5 * MINUTE, - &all, - ); - assert_eq!( - topk.len(), - 3, - "every series is in its job's top 10: {topk:?}" - ); -} - -/// Queries outside the plan, and stored panes that are missing or off the -/// pane grid, are not answered, so the caller forwards them. -#[test] -fn reports_what_it_cannot_answer() { - let mut f = fixture(common::plan("example4b", false)); - let store = &f.precompute.store; - assert!(matches!( - f.query - .bound_raw_sample_windows("up", 70 * MINUTE, store, Default::default()), - Err(Unanswered::NotPlanned(_)) - )); - assert!(matches!( - f.query - .bound_raw_sample_windows(P99, 70 * MINUTE, store, Default::default()), - Err(Unanswered::MissingPane(_)) - )); - let all = samples(&[("latency_ms", &[])], MINUTE, 75 * MINUTE, |_, t| t as f64); - f.precompute.ingest(&all); - let store = &f.precompute.store; - assert!(f - .query - .bound_raw_sample_windows(P99, 70 * MINUTE, store, Default::default()) - .is_ok()); - assert!(matches!( - f.query - .bound_raw_sample_windows(P99, 70 * MINUTE + 1, store, Default::default()), - Err(Unanswered::MissingPane(_)) - )); - // The pane ending at 10 min was skipped as possibly partial. - assert!(matches!( - f.query - .bound_raw_sample_windows(P99, 60 * MINUTE, store, Default::default()), - Err(Unanswered::MissingPane(_)) - )); -} - -/// Evaluation times at the ends of `i64` are not answered and do not panic: -/// binding a window or pane to them would overflow. -#[test] -fn extreme_evaluation_times_are_not_answered() { - let f = fixture(common::plan("example4b", true)); - for time_ms in [i64::MIN, i64::MIN + 1] { - assert!(matches!( - f.query - .bound_raw_sample_windows(P99, time_ms, &f.precompute.store, Default::default()), - Err(Unanswered::Unsupported(_)) - )); - } - // The window ending at `i64::MAX` fits; binding it does not panic. - assert!(f - .query - .bound_raw_sample_windows(P99, i64::MAX, &f.precompute.store, Default::default()) - .is_ok()); - let panes = fixture(common::plan("example4b", false)); - assert!(matches!( - panes.query.bound_raw_sample_windows( - P99, - i64::MIN, - &panes.precompute.store, - Default::default() - ), - Err(Unanswered::Unsupported(_)) - )); -} - -/// Every query of examples 1, 3b and 4b, with and without raw data, answers -/// exactly like the Planner's query-time reference for the same plan -/// (nothing materialized) over the same samples. -#[test] -fn examples_answer_like_the_query_time_reference() { - let series: &[(&str, &[(&str, &str)])] = &[ - ("latency_ms", &[("job", "a")]), - ("latency_ms", &[("job", "b")]), - ("http_requests_total", &[("job", "a"), ("instance", "1")]), - ("http_requests_total", &[("job", "a"), ("instance", "2")]), - ("http_requests_total", &[("job", "b"), ("instance", "1")]), - ]; - // Samples through 70:45: the newest closed pane of every example ends - // at 70 min, the evaluation time. - let all = samples(series, MINUTE, 71 * MINUTE, |i, t| { - (i * 1000) as f64 + ((t / 15_000) % 240) as f64 - }); - for (example, raw) in [ - ("example1", true), - ("example3b", true), - ("example3b", false), - ("example4b", true), - ("example4b", false), - ] { - let plan = common::plan(example, raw); - let reference = plan.query_time_reference().unwrap(); - let mut installed = fixture(plan.clone()); - installed.precompute.ingest(&all); - let mut by_reference = fixture(reference); - for query in plan.queries.iter().map(|q| q.query.as_str()) { - let expected = answer(&mut by_reference, query, 70 * MINUTE, &all); - assert!(!expected.is_empty(), "{example} raw={raw} {query}"); - assert_eq!( - answer(&mut installed, query, 70 * MINUTE, &all), - expected, - "{example} raw={raw} {query}" - ); - } - } -} - -/// A raw window's label matchers go into the fetch, so Prometheus returns -/// only the series the plan keeps. -#[test] -fn bound_raw_sample_windows_carry_the_scans_label_matchers() { - let mut workload = common::workload("example4b"); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = - asap_executor::planner::workload::Query( - "quantile_over_time(0.99, latency_ms{job=\"a\"}[60m])".into(), - ); - let plan = control_plane::plan_workload(&workload, &common::inputs(true)).unwrap(); - let f = fixture(plan); - let ranges = f - .query - .bound_raw_sample_windows( - "quantile_over_time(0.99, latency_ms{job=\"a\"}[60m])", - 70 * MINUTE, - &f.precompute.store, - Default::default(), - ) - .unwrap() - .windows; - assert_eq!(ranges.len(), 1); - assert_eq!( - ranges[0].matchers, - vec![LabelMatcher { - name: "job".into(), - op: MatchOp::Equal, - value: "a".into(), - }] - ); -} diff --git a/data_plane/tests/query_run.rs b/data_plane/tests/query_run.rs deleted file mode 100644 index 72ec4e5a..00000000 --- a/data_plane/tests/query_run.rs +++ /dev/null @@ -1,376 +0,0 @@ -//! Stored outputs built in a `QueryRun` (`MatAtQueryKeptAcrossEvaluations`): -//! #509 Example 4's B3, five 1-minute KLL panes built at query time and -//! stored in the SDS for the next evaluations of -//! `quantile_over_time(0.99, latency_ms[5m])` every minute. - -mod common; - -use std::collections::BTreeMap; - -use asap_planner::{InstallablePlan, StoredOutputBuild}; -use data_plane::engine::{EngineHandle, EngineOptions}; -use data_plane::precompute::RawSample; -use data_plane::precompute::SummaryInstance; -use data_plane::query::{ - BoundRawSampleWindow, IngestionProgress, QueryEngine, QueryRunStats, ResultSample, -}; -use data_plane::sds::{DefinitionId, DefinitionKey, Sds, SummaryStore}; - -const MINUTE: i64 = 60_000; -const P99: &str = "quantile_over_time(0.99, latency_ms[5m])"; - -/// A query engine and the SDS store its stored outputs built in a `QueryRun` go to. -struct Fixture { - engine: QueryEngine, - store: SummaryStore, - definition: DefinitionKey, -} - -fn fixture(plan: &InstallablePlan) -> Fixture { - let sds = Sds::from_plan(plan).unwrap(); - assert!( - sds.ingestion_build_order().unwrap().is_empty(), - "nothing is built at ingestion" - ); - let definition = sds - .definitions - .values() - .find(|d| d.built_at == StoredOutputBuild::QueryRun) - .map(|d| d.key()) - .unwrap_or_else(|| DefinitionKey::unconstrained(DefinitionId(String::new()), 0)); - Fixture { - engine: QueryEngine::new(plan, &sds), - store: SummaryStore::new(), - definition, - } -} - -impl Fixture { - /// The ends, in minutes, of the stored stored outputs built in a `QueryRun`. - fn ends(&self) -> Vec { - (0..=30) - .map(|m| m * MINUTE) - .filter(|end| self.store.get_pane(&self.definition, *end).is_some()) - .map(|end| end / MINUTE) - .collect() - } -} - -/// Two series every 15 s in `[0, 30 min]`, the `n`th sample `25n mod 101` -/// (plus 100 for series b): each 5-minute window's p99 falls in an older -/// pane and changes every minute. KLL is exact below k. -fn samples() -> Vec { - let series: Vec<_> = ["a", "b"] - .iter() - .map(|job| { - data_plane::series_ids::intern(&BTreeMap::from([ - ("__name__".to_string(), "latency_ms".to_string()), - ("job".to_string(), job.to_string()), - ])) - }) - .collect(); - (0..=30 * MINUTE) - .step_by(15_000) - .flat_map(|t| { - series.iter().enumerate().map(move |(i, labels)| RawSample { - labels: labels.clone(), - timestamp_ms: t, - value: (i * 100) as f64 + (t / 15_000 * 25 % 101) as f64, - }) - }) - .collect() -} - -fn fetch(ranges: &[BoundRawSampleWindow], all: &[RawSample]) -> BTreeMap> { - ranges - .iter() - .map(|range| { - let samples = all - .iter() - .filter(|s| { - s.labels["__name__"] == range.metric - && range.start_ms < s.timestamp_ms - && s.timestamp_ms <= range.end_ms - }) - .cloned() - .collect(); - (range.node, samples) - }) - .collect() -} - -/// Answers `P99` at `time_ms` with ingestion's watermark at `watermark_ms` -/// (every metric's newest sample there too), returning the answer and the -/// bound raw-sample windows it read. -fn answer_at_watermark( - k: &mut Fixture, - time_ms: i64, - watermark_ms: Option, - all: &[RawSample], -) -> (Vec, Vec) { - let progress = IngestionProgress { - watermark_ms, - newest_sample_ms: watermark_ms - .map(|w| BTreeMap::from([("latency_ms".to_string(), w)])) - .unwrap_or_default(), - }; - let prepared = k - .engine - .bound_raw_sample_windows(P99, time_ms, &k.store, progress) - .unwrap(); - let ranges = prepared.windows.clone(); - let mut result = k - .engine - .evaluate( - prepared.root, - time_ms, - &mut k.store, - &prepared.progress, - &fetch(&ranges, all), - ) - .unwrap(); - result.sort_by(|a, b| a.labels.cmp(&b.labels)); - (result, ranges) -} - -/// Answers `P99` at `time_ms` once every sample through `time_ms` has -/// arrived: the watermark is `time_ms`. -fn answer( - k: &mut Fixture, - time_ms: i64, - all: &[RawSample], -) -> (Vec, Vec) { - answer_at_watermark(k, time_ms, Some(time_ms), all) -} - -fn spans(ranges: &[BoundRawSampleWindow]) -> Vec<(i64, i64)> { - ranges - .iter() - .map(|r| (r.start_ms / MINUTE, r.end_ms / MINUTE)) - .collect() -} - -/// The installed B3 plan and the Planner's reference for it: the same -/// selection with nothing materialized. -fn plans() -> (InstallablePlan, InstallablePlan) { - let plan = common::plan("example4-b3", true); - let reference = plan.query_time_reference().unwrap(); - (plan, reference) -} - -/// The first evaluation builds all five summaries from one fetch and -/// answers like the reference; the next one, a minute later, fetches only -/// the newest minute, builds one summary, reuses four, and still answers -/// like the reference. -#[test] -fn query_run_outputs_answer_like_the_reference_and_are_reused() { - let (plan, reference_plan) = plans(); - let all = samples(); - let mut k = fixture(&plan); - let mut reference = fixture(&reference_plan); - - let (first, ranges) = answer(&mut k, 10 * MINUTE, &all); - assert_eq!( - spans(&ranges), - vec![(5, 10)], - "one fetch for five summaries" - ); - assert_eq!(first, answer(&mut reference, 10 * MINUTE, &all).0); - assert_eq!(first.len(), 2); - assert_eq!( - k.engine.query_run_stats, - QueryRunStats { - built: 5, - stored: 5, - stored_while_metric_behind: 0, - reused: 0 - } - ); - - let (second, ranges) = answer(&mut k, 11 * MINUTE, &all); - assert_eq!(spans(&ranges), vec![(10, 11)], "only the newest summary"); - assert_eq!(second, answer(&mut reference, 11 * MINUTE, &all).0); - assert_ne!(second, first, "the answer moves with the window"); - assert_eq!( - ( - k.engine.query_run_stats.built, - k.engine.query_run_stats.reused - ), - (6, 4) - ); - - // The same evaluation again builds nothing. - let (again, ranges) = answer(&mut k, 11 * MINUTE, &all); - assert!(ranges.is_empty()); - assert_eq!(again, second); - assert_eq!( - ( - k.engine.query_run_stats.built, - k.engine.query_run_stats.reused - ), - (6, 9) - ); - assert_eq!( - reference.engine.query_run_stats, - QueryRunStats::default(), - "the reference stores nothing" - ); -} - -/// Stored stored outputs built in a `QueryRun` are evicted by the plan's retention (5 min, -/// the oldest summary a 5-minute window reads), plus one pane, measured -/// from the newest closed coverage at or before the watermark, as for -/// stored outputs built in an `IngestionRun`. A far-future evaluation does not move that -/// point: its summary states are built but not stored, and storing resumes. -#[test] -fn query_run_outputs_are_evicted_by_the_plans_retention() { - let (plan, _) = plans(); - let retention = plan.stored_outputs.iter().map(|o| o.retention_ms).max(); - assert_eq!(retention, Some(5 * MINUTE as u64)); - let all = samples(); - let mut k = fixture(&plan); - answer(&mut k, 10 * MINUTE, &all); - assert_eq!(k.ends(), vec![6, 7, 8, 9, 10]); - answer(&mut k, 11 * MINUTE, &all); - assert_eq!(k.ends(), vec![6, 7, 8, 9, 10, 11]); - answer(&mut k, 12 * MINUTE, &all); - assert_eq!(k.ends(), vec![7, 8, 9, 10, 11, 12]); - // An evaluation far ahead of the watermark is answered, but its - // summaries' coverages are not closed, so nothing is stored or evicted. - let (far, ranges) = answer_at_watermark(&mut k, 20 * MINUTE, Some(12 * MINUTE), &all); - assert_eq!(far.len(), 2); - assert_eq!(spans(&ranges), vec![(15, 20)]); - assert_eq!(k.ends(), vec![7, 8, 9, 10, 11, 12]); - // Keeping resumes with the watermark. - answer(&mut k, 13 * MINUTE, &all); - assert_eq!(k.ends(), vec![8, 9, 10, 11, 12, 13]); -} - -/// A summary state is stored only once its coverage is closed (its end at or -/// before the watermark). One read before then is built for that -/// evaluation but not stored, so a later evaluation builds it again and -/// includes a sample that arrived late. -#[test] -fn query_run_outputs_are_stored_only_once_closed() { - let (plan, reference_plan) = plans(); - let mut all = samples(); - let mut k = fixture(&plan); - let mut reference = fixture(&reference_plan); - - // Watermark at 9.5 min: the summary ending at 10 min is still open. - let (first, ranges) = answer_at_watermark(&mut k, 10 * MINUTE, Some(570_000), &all); - assert_eq!(spans(&ranges), vec![(5, 10)]); - assert_eq!(first, answer(&mut reference, 10 * MINUTE, &all).0); - assert_eq!( - k.ends(), - vec![6, 7, 8, 9], - "the open summary state is not stored" - ); - - // A late sample for the open coverage arrives before it closes. - let late = RawSample { - timestamp_ms: 10 * MINUTE - 1_000, - value: 1_000.0, - ..all.iter().find(|s| s.labels["job"] == "a").unwrap().clone() - }; - all.push(late); - let (again, ranges) = answer_at_watermark(&mut k, 10 * MINUTE, Some(585_000), &all); - assert_eq!( - spans(&ranges), - vec![(9, 10)], - "the open summary is built again" - ); - assert_eq!(again, answer(&mut reference, 10 * MINUTE, &all).0); - assert_ne!(again, first, "the late sample is included"); - - // Once the watermark reaches 10 min, the summary state is stored. - answer_at_watermark(&mut k, 10 * MINUTE, Some(10 * MINUTE), &all); - assert_eq!(k.ends(), vec![6, 7, 8, 9, 10]); - // Without any sample there is no watermark, and nothing is stored. - let mut fresh = fixture(&plan); - answer_at_watermark(&mut fresh, 10 * MINUTE, None, &all); - assert!(fresh.store.is_empty()); -} - -/// The store decision uses the watermark when the evaluation chose what to -/// fetch, not when it runs: ingestion that closes a coverage after the -/// fetch was chosen does not make the partial summary built from that -/// fetch permanent. -#[tokio::test] -async fn stores_by_the_watermark_the_fetch_was_chosen_at() { - let (plan, _) = plans(); - let options = EngineOptions { - precompute: data_plane::precompute::PrecomputeOptions { - allowed_lateness_ms: 0, - ..Default::default() - }, - ..EngineOptions::default() - }; - let (engine, _) = EngineHandle::spawn(serde_json::to_string(&plan).unwrap(), options).unwrap(); - let all = samples(); - let through = |t: i64| -> Vec { - all.iter() - .filter(|s| s.timestamp_ms <= t) - .cloned() - .collect() - }; - let settle = |engine: &EngineHandle| { - let engine = engine.clone(); - async move { - while engine.queued_samples() > 0 { - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - } - } - }; - engine.try_ingest(through(570_000)).unwrap(); - settle(&engine).await; - let prepared = engine - .bound_raw_sample_windows(P99.into(), 10 * MINUTE) - .await - .unwrap(); - assert_eq!(prepared.progress.watermark_ms, Some(570_000)); - // The fetch sees only what had arrived; then ingestion closes 10 min. - let raw = fetch(&prepared.windows, &through(570_000)); - engine.try_ingest(through(11 * MINUTE)).unwrap(); - settle(&engine).await; - engine.evaluate(prepared, 10 * MINUTE, raw).await.unwrap(); - assert_eq!(engine.query_run_stats().stored, 4, "6..9 stored, 10 not"); - // So the next evaluation builds the summary ending at 10 min again. - let prepared = engine - .bound_raw_sample_windows(P99.into(), 10 * MINUTE) - .await - .unwrap(); - assert_eq!(spans(&prepared.windows), vec![(9, 10)]); -} - -/// A summary state stored while its own metric's newest sample has not reached its -/// end (the global watermark moved on another metric) is counted. -#[test] -fn counts_outputs_stored_while_their_metric_is_behind() { - let (plan, _) = plans(); - let all = samples(); - let mut k = fixture(&plan); - let progress = IngestionProgress { - watermark_ms: Some(10 * MINUTE), - newest_sample_ms: BTreeMap::from([("latency_ms".to_string(), 8 * MINUTE)]), - }; - let prepared = k - .engine - .bound_raw_sample_windows(P99, 10 * MINUTE, &k.store, progress) - .unwrap(); - let raw = fetch(&prepared.windows, &all); - k.engine - .evaluate( - prepared.root, - 10 * MINUTE, - &mut k.store, - &prepared.progress, - &raw, - ) - .unwrap(); - assert_eq!(k.engine.query_run_stats.stored, 5); - assert_eq!( - k.engine.query_run_stats.stored_while_metric_behind, 2, - "9 and 10" - ); -} diff --git a/data_plane/tests/sds.rs b/data_plane/tests/sds.rs deleted file mode 100644 index 375e183c..00000000 --- a/data_plane/tests/sds.rs +++ /dev/null @@ -1,272 +0,0 @@ -mod common; - -use asap_executor::planner::ir::{NonASAPOp, Operator}; -use asap_planner::StoredOutputBuild; -use common::plan; -use data_plane::sds::{ - DefinitionId, DefinitionKey, Sds, SdsError, Selection, SummaryInstanceKey, SummaryStore, -}; - -/// Example 4b's six ingestion-time KLL panes are one definition with a -/// 10-minute pane; each output reads the pane `i * 10min` before evaluation, -/// and the definition is kept for the largest retention among them. -#[test] -fn example4b_panes_share_one_definition() { - let plan = plan("example4b", false); - let sds = Sds::from_plan(&plan).unwrap(); - assert_eq!(sds.definitions.len(), 1); - let definition = sds.definitions.values().next().unwrap(); - assert_eq!(definition.pane_ms, 600_000); - let mut offsets: Vec = sds - .outputs - .values() - .map(|(id, binding)| { - assert_eq!(id, &definition.key()); - binding.offset_ms - }) - .collect(); - offsets.sort(); - assert_eq!(offsets, (0..6).map(|i| i * 600_000).collect::>()); - let largest = plan.stored_outputs.iter().map(|o| o.retention_ms).max(); - assert_eq!(Some(definition.retention_ms as u64), largest); - let raw = plan - .dag - .dag - .nodes - .iter() - .find(|n| Some(n.id as u64) == definition.raw_input()) - .unwrap(); - assert!(matches!( - raw.payload, - Operator::NonASAP(NonASAPOp::TimeRange { .. }) - )); -} - -/// A definition is kept for the largest retention among its outputs, not -/// its first output's. -#[test] -fn retention_is_the_largest_among_outputs() { - let mut plan = plan("example4b", false); - let last = plan.stored_outputs.len() - 1; - plan.stored_outputs[last].retention_ms = 7 * 3_600_000; - let sds = Sds::from_plan(&plan).unwrap(); - let definition = sds.definitions.values().next().unwrap(); - assert_eq!(definition.retention_ms, 7 * 3_600_000); -} - -/// A plan computed entirely at query time stores nothing. -#[test] -fn query_time_plans_have_no_definitions() { - let plan = plan("example1", true); - assert_eq!(Sds::from_plan(&plan).unwrap(), Sds::default()); -} - -/// A definition's identity is the plan's definition id, the same for every -/// output whose selection differs only in time. -#[test] -fn identity_is_the_plans_definition_id() { - let plan = plan("example4b", false); - let sds = Sds::from_plan(&plan).unwrap(); - let definition = sds.definitions.values().next().unwrap(); - for output in &plan.stored_outputs { - assert_eq!(definition.id.0, output.coverage.definition_id); - } - assert_eq!(&*definition.labels, "", "no label constraints"); -} - -/// Outputs with the same definition id but other label constraints in -/// their selection are other definitions, and their summary instances other -/// keys: the plan's definition id leaves the selection out. -#[test] -fn label_constraints_separate_definitions_of_one_id() { - let mut json = serde_json::to_value(common::workload("example4b")).unwrap(); - let queries = json["query_workload"]["repeating_queries"] - .as_array_mut() - .unwrap(); - let mut other = queries[0].clone(); - queries[0]["query"] = "quantile_over_time(0.99, latency_ms{job=\"a\"}[60m])".into(); - other["query"] = "quantile_over_time(0.99, latency_ms{job=\"b\"}[60m])".into(); - queries.push(other); - let workload = serde_json::from_value(json).unwrap(); - let plan = control_plane::plan_workload(&workload, &common::inputs(false)).unwrap(); - let sds = Sds::from_plan(&plan).unwrap(); - assert_eq!(sds.definitions.len(), 2, "{:?}", sds.definitions.keys()); - let keys: Vec<&DefinitionKey> = sds.definitions.keys().collect(); - assert_eq!(keys[0].id, keys[1].id, "one definition id"); - assert_ne!(keys[0].labels, keys[1].labels); - let mut store = SummaryStore::new(); - for (i, key) in keys.iter().enumerate() { - let selection = sds.definitions[*key].selection_ending_at(600_000); - store.insert(SummaryInstanceKey::new(key.id.clone(), selection), i); - } - assert_eq!(store.get_pane(keys[0], 600_000), Some(&0)); - assert_eq!(store.get_pane(keys[1], 600_000), Some(&1)); -} - -/// Stored outputs built in a `QueryRun` are stored summaries too: their -/// definition says so, and no ingestion run builds them. -#[test] -fn query_run_outputs_have_query_run_definitions() { - let mut plan = plan("example4b", false); - for output in &mut plan.stored_outputs { - output.built_at = StoredOutputBuild::QueryRun; - } - let sds = Sds::from_plan(&plan).unwrap(); - assert_eq!(sds.definitions.len(), 1); - let definition = sds.definitions.values().next().unwrap(); - assert_eq!(definition.built_at, StoredOutputBuild::QueryRun); - assert_eq!(sds.outputs.len(), 6); - assert!(sds.ingestion_build_order().unwrap().is_empty()); -} - -/// A definition shared by an `IngestionRun` and a `QueryRun` output is built -/// at ingestion. -#[test] -fn ingestion_wins_when_outputs_share_a_definition() { - let mut plan = plan("example4b", false); - plan.stored_outputs[0].built_at = StoredOutputBuild::QueryRun; - let sds = Sds::from_plan(&plan).unwrap(); - let definition = sds.definitions.values().next().unwrap(); - assert_eq!(definition.built_at, StoredOutputBuild::IngestionRun); - assert_eq!(sds.ingestion_build_order().unwrap(), vec![definition.key()]); -} - -/// Every stored output reads raw samples today, so no definition has stored -/// inputs; a definition that read itself would be a cycle. -#[test] -fn build_order_follows_stored_inputs() { - let mut sds = Sds::from_plan(&plan("example4b", false)).unwrap(); - let definition = sds.definitions.values_mut().next().unwrap(); - assert!(definition.stored_inputs.is_empty()); - let id = definition.key(); - definition.stored_inputs = vec![id.clone()]; - assert_eq!(sds.ingestion_build_order(), Err(SdsError::Cycle(id))); -} - -/// Panes are found by definition, width and end, and evicted once no query -/// needs them. Panes of one definition id with two widths (`a`'s 60 and 30) -/// are kept apart: neither is found or evicted as the other. -#[test] -fn store_finds_and_evicts_panes() { - let (a, a30, b) = ( - DefinitionKey::unconstrained(DefinitionId("a".into()), 60), - DefinitionKey::unconstrained(DefinitionId("a".into()), 30), - DefinitionKey::unconstrained(DefinitionId("b".into()), 60), - ); - let mut store = SummaryStore::new(); - for end in [60, 120, 180] { - store.insert( - SummaryInstanceKey::new(a.id.clone(), a.selection_ending_at(end)), - format!("a{end}"), - ); - } - store.insert( - SummaryInstanceKey::new(a30.id.clone(), a30.selection_ending_at(120)), - "a30-120".to_string(), - ); - store.insert( - SummaryInstanceKey::new(b.id.clone(), Selection::ending_at(60, 60)), - "b60".to_string(), - ); - assert_eq!(store.get_pane(&a, 120).map(String::as_str), Some("a120")); - assert_eq!( - store.get_pane(&a30, 120).map(String::as_str), - Some("a30-120") - ); - assert_eq!(store.get_pane(&a30, 60), None); - assert_eq!(store.get_pane(&a, 90), None); - store.evict_before(&a.id, &a.selection_ending_at(180)); - assert_eq!(store.get_pane(&a, 120), None); - assert_eq!(store.get_pane(&a, 180).map(String::as_str), Some("a180")); - assert_eq!( - store.get_pane(&a30, 120).map(String::as_str), - Some("a30-120") - ); - assert_eq!(store.get_pane(&b, 60).map(String::as_str), Some("b60")); - assert_eq!(store.len(), 3); -} - -/// `definition` under another identity, reading `stored_inputs`. -fn renamed( - definition: &data_plane::sds::SummaryDefinition, - name: &str, - stored_inputs: &[&str], -) -> data_plane::sds::SummaryDefinition { - let mut definition = definition.clone(); - definition.id = DefinitionId(name.into()); - definition.stored_inputs = stored_inputs - .iter() - .map(|input| { - DefinitionKey::unconstrained(DefinitionId(input.to_string()), definition.pane_ms) - }) - .collect(); - definition -} - -/// Several definitions are ordered after the ones they read, whatever -/// their identities' order: a rollup `a` over `c` and `b`, `b` over `c`. -#[test] -fn build_order_puts_inputs_first() { - let base = Sds::from_plan(&plan("example4b", false)).unwrap(); - let template = base.definitions.values().next().unwrap(); - let mut sds = Sds::default(); - for definition in [ - renamed(template, "a", &["c", "b"]), - renamed(template, "b", &["c"]), - renamed(template, "c", &[]), - ] { - sds.definitions.insert(definition.key(), definition); - } - let order: Vec = sds - .ingestion_build_order() - .unwrap() - .into_iter() - .map(|key| key.id.0) - .collect(); - assert_eq!(order, ["c", "b", "a"]); -} - -/// An `IngestionRun` definition cannot read a `QueryRun` one: that summary -/// is only built when an evaluation needs it. -#[test] -fn build_order_refuses_ingestion_reading_query_run() { - let base = Sds::from_plan(&plan("example4b", false)).unwrap(); - let template = base.definitions.values().next().unwrap(); - let mut query_run = renamed(template, "q", &[]); - query_run.built_at = StoredOutputBuild::QueryRun; - let mut sds = Sds::default(); - for definition in [renamed(template, "i", &["q"]), query_run] { - sds.definitions.insert(definition.key(), definition); - } - assert_eq!( - sds.ingestion_build_order(), - Err(SdsError::IngestionReadsQueryRun { - definition: DefinitionKey::unconstrained(DefinitionId("i".into()), template.pane_ms), - input: DefinitionKey::unconstrained(DefinitionId("q".into()), template.pane_ms), - }) - ); -} - -/// B3's five 1-minute KLL panes, stored outputs built in a `QueryRun` -/// (`MatAtQueryKeptAcrossEvaluations`), are one `QueryRun` definition, each output reading the -/// summary `i * 1min` before evaluation. Nothing is built at ingestion. -#[test] -fn query_run_outputs_share_one_definition() { - let sds = Sds::from_plan(&plan("example4-b3", true)).unwrap(); - assert!(sds.ingestion_build_order().unwrap().is_empty()); - let mut query_run = sds - .definitions - .values() - .filter(|d| d.built_at == StoredOutputBuild::QueryRun); - let definition = query_run.next().unwrap(); - assert!(query_run.next().is_none()); - assert_eq!(definition.pane_ms, 60_000); - let mut offsets: Vec = sds - .outputs - .values() - .filter(|(id, _)| id == &definition.key()) - .map(|(_, binding)| binding.offset_ms) - .collect(); - offsets.sort(); - assert_eq!(offsets, (0..5).map(|i| i * 60_000).collect::>()); -} diff --git a/data_plane/tests/server.rs b/data_plane/tests/server.rs index c87e087b..65f2cf02 100644 --- a/data_plane/tests/server.rs +++ b/data_plane/tests/server.rs @@ -369,11 +369,10 @@ async fn reports_sample_counters() { } assert_eq!(body["engine_running"], true); assert_eq!(body["samples_queued"], 0); - let counters = &body["precompute"]; + let counters = &body["ingestion"]; let total: u64 = [ "samples_accepted", "samples_dropped_late", - "samples_skipped_first_pane", "samples_dropped_future", "samples_unrouted", ] diff --git a/data_plane/tests/workloads/random-mixed-seed-862-01.json b/data_plane/tests/workloads/random-mixed-seed-862-01.json new file mode 100644 index 00000000..2b9c06d1 --- /dev/null +++ b/data_plane/tests/workloads/random-mixed-seed-862-01.json @@ -0,0 +1,237 @@ +{ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + { + "query": "sum by (region, service, job) (sum_over_time(data[10m]))", + "demand": { + "fixed_interval_at": { + "interval": 20000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "sum by (region, service) (sum_over_time(data[10m]))", + "demand": { + "fixed_interval_at": { + "interval": 20000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, sum_over_time(data[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 30000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (8, increase(data_total[10m]))", + "demand": { + "fixed_interval_at": { + "interval": 20000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, rate(data_total[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 30000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} diff --git a/data_plane/tests/workloads/random-mixed-seed-862-02.json b/data_plane/tests/workloads/random-mixed-seed-862-02.json new file mode 100644 index 00000000..c314736c --- /dev/null +++ b/data_plane/tests/workloads/random-mixed-seed-862-02.json @@ -0,0 +1,237 @@ +{ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + { + "query": "sum by (region, service, job) (sum_over_time(data[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 5000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "sum by (region, service) (sum_over_time(data[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 5000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, sum_over_time(data[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, increase(data_total[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 5000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, rate(data_total[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} diff --git a/data_plane/tests/workloads/random-mixed-seed-862-03.json b/data_plane/tests/workloads/random-mixed-seed-862-03.json new file mode 100644 index 00000000..d290502c --- /dev/null +++ b/data_plane/tests/workloads/random-mixed-seed-862-03.json @@ -0,0 +1,237 @@ +{ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + { + "query": "sum by (region, service, job) (sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "sum by (region, service) (sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service) (quantile_over_time(0.90, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service, job) (quantile_over_time(0.90, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, sum_over_time(data[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 30000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (8, increase(data_total[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, rate(data_total[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 30000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} diff --git a/data_plane/tests/workloads/random-mixed-seed-862-04.json b/data_plane/tests/workloads/random-mixed-seed-862-04.json new file mode 100644 index 00000000..5bea9d20 --- /dev/null +++ b/data_plane/tests/workloads/random-mixed-seed-862-04.json @@ -0,0 +1,237 @@ +{ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + { + "query": "sum by (region, service, job) (sum_over_time(data[10m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "sum by (region, service) (sum_over_time(data[10m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 2000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, sum_over_time(data[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 30000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, increase(data_total[10m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (8, rate(data_total[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 30000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} diff --git a/data_plane/tests/workloads/random-mixed-seed-862-05.json b/data_plane/tests/workloads/random-mixed-seed-862-05.json new file mode 100644 index 00000000..7d5bf7d6 --- /dev/null +++ b/data_plane/tests/workloads/random-mixed-seed-862-05.json @@ -0,0 +1,237 @@ +{ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + { + "query": "sum by (region, service, job) (sum_over_time(data[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "sum by (region, service) (sum_over_time(data[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (8, sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (4, sum_over_time(data[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (8, increase(data_total[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region) (8, rate(data_total[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} diff --git a/data_plane/tests/workloads/random-mixed-seed-862-06.json b/data_plane/tests/workloads/random-mixed-seed-862-06.json new file mode 100644 index 00000000..772bb4c3 --- /dev/null +++ b/data_plane/tests/workloads/random-mixed-seed-862-06.json @@ -0,0 +1,237 @@ +{ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + { + "query": "sum by (region, service, job) (sum_over_time(data[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 5000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "sum by (region, service) (sum_over_time(data[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 5000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region, service) (quantile_over_time(0.50, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "demand": { + "fixed_interval_at": { + "interval": 1000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, sum_over_time(data[1m]))", + "demand": { + "fixed_interval_at": { + "interval": 2000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (8, sum_over_time(data[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (8, increase(data_total[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 5000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + }, + { + "query": "topk by (region, service) (4, rate(data_total[20m]))", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.050000000000000044, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} diff --git a/data_plane/tests/workloads/sketch-plan-quantile-grouped.json b/data_plane/tests/workloads/sketch-plan-quantile-grouped.json new file mode 100644 index 00000000..2c06a1ef --- /dev/null +++ b/data_plane/tests/workloads/sketch-plan-quantile-grouped.json @@ -0,0 +1,59 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "avg by (region) (quantile_over_time(0.9, data[5m]))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.05, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + } + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} \ No newline at end of file diff --git a/data_plane/tests/workloads/sketch-plan-quantile-single.json b/data_plane/tests/workloads/sketch-plan-quantile-single.json new file mode 100644 index 00000000..0b1c22d8 --- /dev/null +++ b/data_plane/tests/workloads/sketch-plan-quantile-single.json @@ -0,0 +1,59 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "quantile_over_time(0.99, data[5m])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.05, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + } + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} \ No newline at end of file diff --git a/data_plane/tests/workloads/sketch-plan-ratio-of-quantiles-1h.json b/data_plane/tests/workloads/sketch-plan-ratio-of-quantiles-1h.json new file mode 100644 index 00000000..9ca6f344 --- /dev/null +++ b/data_plane/tests/workloads/sketch-plan-ratio-of-quantiles-1h.json @@ -0,0 +1,45 @@ +{ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + { + "query": "quantile_over_time(0.9, data[1h]) / quantile_over_time(0.5, data[1h])", + "demand": { + "fixed_interval_at": { + "interval": 60000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.05, + "failure_probability": 0.01, + "accuracy_scope": "whole_result" + }, + "max_latency_ms": null + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 2000000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 5 + } + } + } +} \ No newline at end of file diff --git a/data_plane/tests/workloads/sketch-plan-topk-rate.json b/data_plane/tests/workloads/sketch-plan-topk-rate.json new file mode 100644 index 00000000..400b7472 --- /dev/null +++ b/data_plane/tests/workloads/sketch-plan-topk-rate.json @@ -0,0 +1,59 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "topk(5, sum by (job) (rate(data_total[5m])))", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": { + "accuracy": { + "max_error": 0.05, + "failure_probability": 0.3, + "accuracy_scope": "whole_result" + } + }, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 200000.0 + } + }, + "sources": { + "data": { + "series_count": 10000, + "metric_type": "gauge", + "column_signs": {"value": "non_negative"}, + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "data_total": { + "series_count": 10000, + "metric_type": "counter", + "sample_interval": 100, + "stream_norms": null, + "max_distinct_per_evaluation": null + }, + "events": { + "series_count": 10000, + "metric_type": null, + "sample_interval": null, + "stream_norms": null, + "max_distinct_per_evaluation": 10000 + } + } + } +} \ No newline at end of file diff --git a/examples/planner-layering-example1.json b/examples/planner-layering-example1.json index 0e5a68df..1384d03b 100644 --- a/examples/planner-layering-example1.json +++ b/examples/planner-layering-example1.json @@ -6,7 +6,7 @@ { "query": "sum by (job) (rate(http_requests_total[1m]))", "demand": { - "fixed_interval": 10000 + "fixed_interval_at": {"interval": 10000, "evaluation_phase": 0} }, "requirements": { "accuracy": { @@ -25,7 +25,7 @@ { "query": "topk by (job) (10, sum_over_time(http_requests_total[1m]))", "demand": { - "fixed_interval": 10000 + "fixed_interval_at": {"interval": 10000, "evaluation_phase": 0} }, "requirements": { "accuracy": { diff --git a/examples/planner-layering-example3b.json b/examples/planner-layering-example3b.json index c943e415..7c07d8b7 100644 --- a/examples/planner-layering-example3b.json +++ b/examples/planner-layering-example3b.json @@ -6,7 +6,7 @@ { "query": "quantile_over_time(0.99, latency_ms[5m])", "demand": { - "fixed_interval": 60000 + "fixed_interval_at": {"interval": 60000, "evaluation_phase": 0} }, "requirements": { "accuracy": { diff --git a/examples/planner-layering-example4-b3.json b/examples/planner-layering-example4-b3.json index 9ad9481d..c6dab44b 100644 --- a/examples/planner-layering-example4-b3.json +++ b/examples/planner-layering-example4-b3.json @@ -6,7 +6,7 @@ { "query": "quantile_over_time(0.99, latency_ms[5m])", "demand": { - "fixed_interval": 60000 + "fixed_interval_at": {"interval": 60000, "evaluation_phase": 0} }, "requirements": { "accuracy": { diff --git a/examples/planner-layering-example4b.json b/examples/planner-layering-example4b.json index ab13a77f..4e0b3247 100644 --- a/examples/planner-layering-example4b.json +++ b/examples/planner-layering-example4b.json @@ -6,7 +6,7 @@ { "query": "quantile_over_time(0.99, latency_ms[60m])", "demand": { - "fixed_interval": 600000 + "fixed_interval_at": {"interval": 600000, "evaluation_phase": 0} }, "requirements": { "accuracy": {