diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 8ef34de7d..dc80a7492 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -107,6 +107,19 @@ exclude_re = [ # (`entry_key`): its error is discarded (`.ok()`), so its message is never # read at all. 'stack-encrypt/src/dynamic/record\.rs:\d+:\d+: replace entry_key:: for Keys>::deserialize:: for V>::expecting -> fmt::Result with Ok\(Default::default\(\)\)$', + # The JSON index. `unhex` joins a high nibble shifted into the top four + # bits with a low nibble below 16: the bits never overlap, so `^` is `|`. + 'stack-encrypt/src/json/mod\.rs:\d+:\d+: replace \| with \^ in unhex$', + # `JsonPath::root` is `Self::default()`. + 'stack-encrypt/src/json/path\.rs:\d+:\d+: replace JsonPath::root -> Self with Default::default\(\)$', + # The sealed root's `None` token: with that arm gone, the root falls to + # the arm that wraps its parent's token, and the root has no parent, so + # it is `None` there too. `the_sealed_root_has_no_token_and_its_fields_are_heads` + # pins the value; the arm stays so the match says it. + 'stack-encrypt/src/json/derive\.rs:\d+:\d+: delete match arm \(0, true\) in Walker<\x27_>::visit$', + # A document always has a root, so its terms are never empty: `false` is + # the value `is_empty` returns. The `true` replacement is still caught. + 'stack-encrypt/src/json/derive\.rs:\d+:\d+: replace JsonTerms::is_empty -> bool with false$', ] # Headroom over the measured baseline before a slow-but-correct mutant is diff --git a/.changeset/eql-json-tree-domain.md b/.changeset/eql-json-tree-domain.md new file mode 100644 index 000000000..1616779ff --- /dev/null +++ b/.changeset/eql-json-tree-domain.md @@ -0,0 +1,7 @@ +--- +"@cipherstash/eql": minor +--- + +**EQL v4 gains the `json_tree` domain family: `public.eql_v4_json_tree`, an encrypted JSON document stored as one entry per node, with `@>` and `<@` that agree with PostgreSQL's own `jsonb @>` across array elements.** The family is new beside `public.eql_v4_json` and is not a port of the v3 SteVec surface: it is what Stack Encrypt's JSON index writes. A document's entries each carry a path selector `s` and a value selector `vs` (containment is a set comparison over the pair), the node's own ciphertext `c` (its label, kind and scalar value; a document's ciphertext is linear in its size, where a SteVec document repeats every subtree at every level), and a token chain (`p`, `w`, `o`) that lets core PostgreSQL walk the subtree under a node from one client-derived token: `eql_v4.json_tree_subtree(doc, token)`, `json_tree_subtree_at(doc, token, selector)`, `json_tree_children` and `json_tree_child_count`. By default the root is sealed, so no query can hand the server a token for the whole column. Containment projects a stored document into an element-nested shape from its group tags (`n`, `g`), `eql_v4.json_tree_query(doc)`, and compares it with the needle (`eql_v4.query_json_tree`) by native `jsonb @>`, so each needle array element must fit inside one stored element, which closes the cross-element false positives of selector-set containment (#1231) for this family; a GIN index on `(eql_v4.json_tree_query(col)::jsonb) jsonb_path_ops` serves it. `->` by selector returns an entry that opens on its own (`public.eql_v4_json_tree_entry`, with the header and ordinal grafted), and `eql_v4.query_json_tree_leaf` orders an extracted number or string leaf with `<`, `<=`, `>`, `>=`. Every other native jsonb operator on the column is blocked, as on the other EQL domains. EQL v4 only (`-- ONLY EQL VERSION: 4`); the v3 bundle is unchanged. The `eql-bindings` crate carries the hand-written `v4::json_tree` types with Stack Encrypt's `EncryptFrom` / `DecryptInto` for the document and the entry. + +Why. Stack Encrypt had no JSON index: the operators (`Contains`, `ContainedBy`, `AtPath`) were declared and answered by nothing, and the v4 JSON wire format was unconstrained by anything shipped. This is Phase 1 and 2 of the `json_tree` index's phased plan (#1249), implemented against `docs/plans/2026-10-17-json-tree-index.md`, whose "Implementation notes" section records where the code deviates from the design. diff --git a/.changeset/eql-v4-bindings.md b/.changeset/eql-v4-bindings.md index 7f0d61385..e92cee2ee 100644 --- a/.changeset/eql-v4-bindings.md +++ b/.changeset/eql-v4-bindings.md @@ -2,7 +2,7 @@ '@cipherstash/eql': minor --- -**The `eql-bindings` crate has an `eql_bindings::v4` module, with the `stack-encrypt` feature: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt, so the module exists only with that feature and its types derive Stack Encrypt's traits without a further gate. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch` (3-grams, `k = 6`, a 2048-bit filter, pinned as `eql_bindings::encryption::EqlMatch`; text shorter than three characters is refused, stored or queried, and only `stored @@ query` matches until #1171 is fixed), from the Rust primitives `i32`, `i64`, `bool`, `f32` and `f64`; the Go SDK's `encrypt/eql` table marks them producible. Stack Encrypt also produces the number equality types `IntegerEq`, `BigintEq`, `RealEq` and `DoubleEq`, whose `=` compares as PostgreSQL compares the plain types: an integer has one term whatever its width, and for floats `-0.0 = 0.0` and `NaN = NaN` (ADR-0002's semantic baseline). Every type it cannot produce yet names the issue that tracks its blocker. The by-name target entry points (`eql_bindings::encryption::targets::encrypt`, `decrypt` and `query`) exchange the plaintext as Stack Encrypt's `dynamic::ValueFrame`, one value as its transport frame, rather than a value tree. They, and the generated `v4::targets` table, need the new `targets` feature, which turns on `stack-encrypt` and Stack Encrypt's `dynamic` feature. `stack-encrypt` alone no longer builds Stack Encrypt's `dynamic` layer, so a Rust program using the typed v4 types through `encrypt_as` doesn't compile it. +**The `eql-bindings` crate has an `eql_bindings::v4` module, with the `stack-encrypt` feature: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt, so the module exists only with that feature and its types derive Stack Encrypt's traits without a further gate. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch` (3-grams, `k = 6`, a 2048-bit filter, pinned as `eql_bindings::encryption::EqlMatch`; text shorter than three characters is refused, stored or queried, and only `stored @@ query` matches until #1171 is fixed), from the Rust primitives `i32`, `i64`, `bool`, `f32` and `f64`; the Go SDK's `encrypt/eql` table marks them producible. Stack Encrypt also produces the number equality types `IntegerEq`, `BigintEq`, `RealEq` and `DoubleEq`, whose `=` compares as PostgreSQL compares the plain types: an integer has one term whatever its width, and for floats `-0.0 = 0.0` and `NaN = NaN` (ADR-0002's semantic baseline). Every type it cannot produce yet names the issue that tracks its blocker. A Rust caller queries those types through Stack Encrypt's typed query chain: `cipher.query(op::Eq, v).on(&plan)` on a plan for `TextEq` or a number equality type returns the type's query twin, and `cipher.query(op::Matches, text).on(&plan)` on a plan for `TextMatch` returns a `MatchTwin` with `verify` and `target_limit`, the same bytes and answers as the data path the Go SDK uses (`eql_bindings::encryption::query`; an operator a type does not answer does not compile). The by-name target entry points (`eql_bindings::encryption::targets::encrypt`, `decrypt` and `query`) exchange the plaintext as Stack Encrypt's `dynamic::ValueFrame`, one value as its transport frame, rather than a value tree. They, and the generated `v4::targets` table, need the new `targets` feature, which turns on `stack-encrypt` and Stack Encrypt's `dynamic` feature. `stack-encrypt` alone no longer builds Stack Encrypt's `dynamic` layer, so a Rust program using the typed v4 types through `encrypt_as` doesn't compile it. **The EQL v4 SQL bundle has range query domains and `eql_v4.between`, so a `BETWEEN` query is one operand.** The v4 bundle (`cipherstash-encrypt-v4.sql`) is built from this source and reaches users as the `eql-v4-install.sql` attached to each Go release; neither the npm package nor the crate ships it, and the v3 SQL they ship does not change. Every ordered family has one range query domain per order term, beside its single-bound ones: `eql_v4.query_integer_range_ore` beside `eql_v4.query_integer_ord_ore`, `eql_v4.query_integer_range_ope` beside `eql_v4.query_integer_ord_ope`, and the same for `smallint`, `bigint`, `date`, `timestamp`, `numeric`, `text`, `real` and `double`. A range is `{"v": 4, "i": …, "lo": , "hi": }`: each bound is the single-bound domain's `ob` (ORE) or `op` (OPE) value, in a `{v, i, lo, hi}` envelope. Stack Encrypt does not build that envelope yet; it will need to when the ordered types become producible. `eql_v4.between(column, range)` is true when the column's order term lies between the bounds, both included, and has an overload for every ordered column domain with the range of the term it stores (`_ord_ore` and `_search_ore` take the ORE range; `_ord`, `_ord_ope` and `_search` the OPE one). It is an inlinable SQL function, so a btree on the column's order term (`eql_v4.ord_term_ore(col)` or `eql_v4.ord_term(col)`) serves it as one index range scan. Write it schema-qualified (`BETWEEN` is an SQL keyword, so a bare `between(...)` does not parse), and cast the range to its exact domain (`$1::eql_v4.query_integer_range_ore`): an uncast `$1`, a `jsonb` parameter, or a range of another term or family fails with SQLSTATE 42725 ("function … is not unique"), unlike the single-bound operators, which resolve an untyped `$1`. An ORE range scan needs the `ore_block_256` btree operator class, which the installer creates only as a superuser; on an install without it (Supabase and other non-superuser roles), the ORE range domains are disabled like the single-bound ORE domains, and a value cast to one raises `feature_not_supported` naming the family's OPE range. In the crate, `eql_bindings::v4` (`stack-encrypt` feature) has a type for each range domain (`IntegerRangeOreQuery`, `TextRangeOpeQuery`, …, listed by `v4::all_range_query()`). EQL v3 has no range domains: its SQL and types are unchanged (ADR-0009, decision 7). diff --git a/.changeset/eql-v4-block-ore-types.md b/.changeset/eql-v4-block-ore-types.md new file mode 100644 index 000000000..d5cb664f8 --- /dev/null +++ b/.changeset/eql-v4-block-ore-types.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**EQL v4 compares ore-rs's 6-bit block ORE, and Stack Encrypt produces the block ORE types `IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, `DoubleOrdOre`, `TextOrdOre` and `TextSearchOre`.** A v4 `ob` term is one ore-rs ciphertext: the fixed-width scheme for numbers, the chained scheme for text (padded to a power-of-two length bucket of at least 16 bytes). The v4 bundle names its block ORE objects `ore_block_v2` (`eql_v4_internal.ore_block_v2_term`, `compare_ore_block_v2_term`) and compares them with a new comparator that checks each term's shape, raises on malformed or mismatched terms, and runs a constant operation count for terms of one shape. The EQL v3 bundle is unchanged, byte for byte: its `ob` is still cipherstash-client's 8-bit block ORE under `ore_block_256`. See [U-002](../../docs/upgrading/v3.1.md#u-002-the-block-ore-term-is-ore-rss-6-bit-scheme): the comparator refuses the 8-bit terms the previous `eql_v4` bundle compared, so rows or operands holding them must be written again through Stack Encrypt. diff --git a/.changeset/eql-v4-text-search-optional-terms.md b/.changeset/eql-v4-text-search-optional-terms.md new file mode 100644 index 000000000..b9152c801 --- /dev/null +++ b/.changeset/eql-v4-text-search-optional-terms.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**An EQL v4 text search query carries only the terms its operator reads.** `eql_v4.query_text_search` and `eql_v4.query_text_search_ore` now require at least one of their terms (`hm`, `op` or `ob`, `bf`) instead of all of them, so `col = $1` and `col < $1` work on text too short to cut a 3-gram, which has no match term. Each extractor over such a query raises (`invalid_parameter_value`) wherever it reads the operand when the query lacks the term it reads, or holds it as JSON `null` or an empty array, naming the domain, the term and the operators that compare it: `=` without `hm`, `<` without `op`, `@@` without `bf`. So a missing term raises rather than matching no row silently, or every row; the raise is at evaluation, so a generic plan that never reaches the predicate returns no row and no error. Stored `eql_v4_text_search` values still carry every term. The `eql_v3` domains are unchanged: the v3 bundle is byte for byte what it was. With it, **Stack Encrypt produces `TextSearch`** (`eql-bindings`, `stack-encrypt` feature): `TextSearchQuery`'s terms are `Option`s, built per index through `TextSearchEqualityQuery`, `TextSearchOrderQuery` and `TextSearchMatchQuery`, and the stored value carries every term: text too short for a 3-gram stores the empty filter (`"bf": []`, the new `v4::terms::SearchBloomFilter` field type), so `=` and `<` find it and no `@@` query does, where `TextMatch` still refuses it. `TextSearchQuery`'s `Option` terms and `TextSearch`'s `SearchBloomFilter` change the fields of the unreleased `v4` module only; no published `eql-bindings` has it. See [U-001](../../docs/upgrading/v3.1.md#u-001-a-text-search-query-carries-only-the-terms-its-operator-reads) for what a hand-built query must carry, the raise, and the rollback. diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index ead9b325c..e0b1f16be 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -597,6 +597,12 @@ jobs: rustup component add --toolchain "${active_rust_toolchain}" rustfmt clippy mise run test:crates + # The block ORE timing harness's ciphertext generator is a detached + # workspace with its own Cargo.lock that no job builds, so its lock is + # checked here rather than left to drift until someone runs the harness. + - name: Check the timing harness's Cargo.lock + run: cargo metadata --locked --format-version 1 --manifest-path tests/timing/ore_block_v2/ctgen/Cargo.toml > /dev/null + # The three encryption checks are mise tasks rather than raw cargo here, # so `eql-suite-ci.test.mjs` counts them among the cargo tasks a root # workflow must reach, and `eql-workflow-filters.test.mjs` derives the diff --git a/AGENTS.md b/AGENTS.md index 70e6870a5..87388fcc4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -681,6 +681,18 @@ monorepo, which is where the silent failures are. - **Changesets for `@cipherstash/eql` go in the repo-root `.changeset/`.** The subtree's own `.changeset/` was deleted with the import; there is no second one to put them in by mistake. +- **The JSON tree family is EQL v4 only and spans two crates.** The engine + half is `packages/stack-encrypt/src/json/` (the `Json` index, `JsonDocument`, + the query terms); the SQL is `packages/eql/src/v3/json_tree/` under + `-- ONLY EQL VERSION: 4`; the EQL types are hand-written in + `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; and the live + oracle, containment over the encrypted column against PostgreSQL's own + `jsonb @>` over the plaintexts, is `packages/eql/tests/encryption/tests/json_tree.rs`, + run by `mise run test:encryption:postgres`. The design and its + implementation notes are `docs/plans/2026-10-17-json-tree-index.md`. + A change to the derivations on one side (a PAE domain, the header layout, + the projection shape) is a change to the other: the Postgres suite is + what holds them together. ## Working on the Rust crates @@ -695,8 +707,12 @@ exact pin would stop the suite sharing one `cts-common` with it. - **Three Cargo workspaces, not one.** The root workspace (the six stack-* crates and the three node binding crates), protect-ffi's and EQL's. The root `Cargo.toml` excludes the other two, and they pin their own vitaminc. The - three fuzz crates and the two Go guests are detached workspaces too, each - with its own `Cargo.lock`. + three fuzz crates, the two Go guests and the EQL v4 block ORE timing + harness's ciphertext generator + (`packages/eql/tests/timing/ore_block_v2/ctgen`) are detached workspaces + too, each with its own `Cargo.lock`. None of them has a Dependabot entry. + CI builds the generator nowhere, so `test-eql.yml` checks its lock with + `cargo metadata --locked`, and `run.sh` builds it with `--locked`. - **The toolchain is pinned in the root `mise.toml`:** Rust 1.94.1 (the `stack-encrypt` `tests/ui` trybuild snapshots record its diagnostics) and Go 1.26. EQL's `mise.toml` overrides the Rust pin in its folder. diff --git a/Cargo.lock b/Cargo.lock index b14cac340..bc8fce38a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -350,6 +351,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "brotli" version = "8.0.2" @@ -383,6 +390,12 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -2128,6 +2141,20 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2138,6 +2165,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" @@ -2153,6 +2189,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2201,6 +2258,35 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.6", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3274,8 +3360,10 @@ dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", + "sha2 0.11.0", "stack-auth", "stack-encrypt-derive", "stack-kms", @@ -3398,6 +3486,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.114" diff --git a/Cargo.toml b/Cargo.toml index 17bec471a..50f5b3cb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,11 @@ recipher = "=0.3.1" # which Stack Encrypt's order terms are encrypted through. It first ships # in a pre-release, and Cargo resolves a pre-release only when named. cllw-ore = { version = "=0.5.2-rc.1", default-features = false, features = ["vitaminc"] } +# ore-rs's `vitaminc` feature implements the same traits for block ORE: +# `Bit6BlockOre` (fixed width up to ten bytes) and `ChainedBlockOre` +# (variable length), the 6-bit schemes EQL v4's `_ord_ore` domains store. +# Both wire formats are frozen as of 0.9.0. ore-rs feeds only stack-encrypt. +ore-rs = { version = "0.9.0", default-features = false, features = ["vitaminc"] } # External dependencies, with the suite's feature lists. base64 = "0.22.0" diff --git a/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md index 8230b207b..ab2c4fff9 100644 --- a/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md +++ b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md @@ -293,7 +293,9 @@ equality term on the same column still reveals which rows share a timestamp. problem, as it is for a plaintext float column. - **Order:** every scalar kind except `Bool`, under all three schemes, with one exception: block ORE for text, which waits for ore-rs's variable-length - block ORE (see Deferred). + block ORE (see Deferred). *Amended 2026-10-19:* the exception is gone. + ore-rs 0.9.0 released its chained scheme, and Stack Encrypt orders text + with it (see "EQL v4"). - **`Bool` has a ciphertext only.** A keyed hash or an order term over a domain of two values hides nothing: it splits the rows into two groups, and an order term also says which group is `true`. Stack Encrypt stops @@ -443,6 +445,67 @@ without loss, and refuses it otherwise. - **During the overlap**, expected to last a quarter or more, cipherstash-client takes fixes only. New kinds and domains are produced through Stack Encrypt and v4. An exception is a decision written down in its issue. +- *Amended 2026-10-19:* **a v4 search query carries only the terms its + operator reads.** The text search query domains (`eql_v4.query_text_search` + and `eql_v4.query_text_search_ore`, whose terms include the match term + `bf` beside others) require at least one of their terms rather than all + of them. Text shorter than one 3-gram has no match term, so a query that + had to carry every term could not express `=` or `<` on short text. + Stack Encrypt emits only the term of the index that answers the query's + operator. Every extractor over such an operand raises wherever it reads + the operand when the term it reads is absent, JSON `null` or an empty + array, naming the domain, the term and the operators that compare it, and + never the operand's contents, rather than reading it as `NULL` (no row) or + as an empty filter. The raise is at evaluation: a literal operand is + folded and raises at plan time, while a bound parameter under a generic + plan whose predicate is never reached returns no row and no error. A + stored search value carries every term, and text too short for a match + term stores the empty filter (`"bf": []`), the stored side's answer in + Stack Encrypt: `=` and `<` find it, and no `@@` query does, because a + query's filter is never empty (the extractor refuses `"bf": []`). That is + a claim about queries: `@@` between two stored values (a self-join + `a.name @@ b.name`, or `matches(col, )`) keeps + the shared guard, so two stored empty filters match each other there, as + they do in v3. No producer builds that predicate. A match + query of such text is refused, as it is for `TextMatch`, whose stored + value still refuses it too. The v3 domains are cipherstash-client's format + and keep requiring every term, byte for byte, so here the v4 SQL differs + from the v3 SQL. The one source holds both forms between version-block directive lines (`-- {{#v3}}` … `-- {{/v3}}`, + `-- {{#v4}}` … `-- {{/v4}}`), which the renderer keeps for their own + version and drops from the other, so the v3 bundle renders exactly as it + did before the block was written. +- *Amended 2026-10-19:* **a v4 `ob` term is ore-rs's 6-bit block ORE.** The + key name is v3's, but the format is not. Stack Encrypt writes one term per + value, the whole ore-rs ciphertext (left and right halves) as one hex + element: the fixed-width scheme (`Bit6BlockOre`, wire header `02 02`) for + the integers and floats, and the chained scheme (`ChainedBlockOre`, `02 + 03`, then the padding policy) for text, padded with NUL bytes to a + power-of-two bucket of at least 16 bytes. v3's `ob` is cipherstash-client's + 8-bit block ORE, up to six terms per text value. The two need different + comparators, so the v4 SQL holds its own (a version block in + `sem/ore_block_256/functions.sql`), and every name built on the block ORE + type is a placeholder too: `{{ore_block}}` renders as `ore_block_256` in + v3 and `ore_block_v2` (ore-rs's wire version 2) in v4, so no v4 object + carries the 8-bit format's name. The v3 bundle renders byte for byte as + before. + + The v4 comparator checks both terms' shape before it compares (header, + block count, exact length, padding policy, every permuted symbol below 64) + and raises, naming the length and never the bytes, on a malformed term, on + terms of different schemes, on fixed-width terms of different block counts + and on chained terms padded under different policies. For terms of one + shape it runs a constant operation count: one statement per block + evaluates both of its comparisons and latches the first difference + arithmetically, and the hash runs exactly once on every path + (`packages/eql/tests/timing/ore_block_v2/` measures it). + The hash is ore-rs's σ-MMO over AES under a fixed public key, which + pgcrypto computes. A query carries the full ciphertext too, not the left + half alone, because a btree index or `between` can pass the stored value + first, and the comparator reads its second operand's right half. + + A comparison of two chained terms reveals the length of their common + prefix in 6-bit blocks, and a chained term reveals its text's length + bucket. Both are ore-rs's documented leakage for the chained scheme. ### Review @@ -534,9 +597,12 @@ through a wildcard. ## Deferred -- **Block ORE for text** waits for ore-rs's chained, variable-length block ORE +- ~~**Block ORE for text** waits for ore-rs's chained, variable-length block ORE to be reviewed and released, then arrives in a minor release of - `vitaminc-ore`. Until then it is an entry in the exceptions list. + `vitaminc-ore`. Until then it is an entry in the exceptions list.~~ + *Amended 2026-10-19:* done. ore-rs 0.9.0's `vitaminc` feature implements + `OrderScheme` for its chained scheme, and Stack Encrypt produces + `TextOrdOre` and `TextSearchOre` with it (see "EQL v4"). - **Block ORE for `Bool`.** Block ORE stored as right ciphertexts only is fully randomised and semantically secure, so a two-value domain leaks nothing through it. `Bool` could support that scheme alone, enforced by a diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index fb595fb92..3ccbd3ff7 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -332,6 +332,28 @@ Scope: prove the three changes end to end with real encryption under the fake ke Each phase is a PR against `next`, where #1209 landed. +## Implementation notes (2026-10-19, Phases 1 and 2) + +The index, the SQL and the live oracle landed in the pull request stacked on this document; what follows records where the code is and where it departs from the design above, so the two can be read against each other. Phase 0 was folded into Phase 2, and Phases 3 and 4 remain open. + +**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan reads it and refuses to run it (`Reason::JsonIndexPending`). The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus and random documents encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, with the GIN index shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs`, against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. + +**Decisions taken by the code.** Of the eleven listed below: 1, hex for every sixteen-byte field and base64 for `c` and `h`, pending Phase 3; 2, leaf ordering terms bound to the column only, numbers as the `f64` they round to and strings as text, under the scalar `Ope` index's derivation; 3, wildcard arrays only; 4, group tags on, and only on container array elements; 5, the child count is in every container node's ciphertext, and a truncated subtree is `OpenedType`, not a shorter array; 7, plain SHA-256; 8, the option is the boolean `sealed_root`, on by default, recorded in the header's flags; 9, the projection is built from the tags server-side (`eql_v4.json_tree_query`) and `groups_ok` was not needed; 10, position-keyed parent tags on, the ordinal stored as `o`, object members ranked under the parent's token; 11, keyed: a child's token is `PRF(json/token-step ‖ column, PAE(t(parent), 0x01, key))` (`0x02, LE64(index)` for an element) and an object member's rank is `PRF(json/rank ‖ column, PAE(t(parent), key))`, each under its own PAE domain rather than the heads' `json/token/v1`, so a token's holder can neither test key names against the child tokens the walk unwraps nor rank guessed names against the ordinals. Decision 6 is out of scope, as proposed. The first implementation hashed both steps unkeyed, as the token chain section above writes them; it was corrected before any release, so no stored document carries the unkeyed form. + +**Deviations.** + +- *Nonces are random, not derived from the selector and ordinal.* Every entry seals through the engine's one AEAD path (`vitaminc-encrypt`'s AES-256-GCM with its own random 96-bit nonce, carried in `c`), so there is one leaf cipher in the crate and no second AES-GCM call site. Uniqueness under one document key is then probabilistic (`N² / 2^97` at `N` entries) rather than structural. The entry's AAD still binds its ordinal `j`, so a moved or reordered entry fails to open as the design requires; the cost is twelve bytes per entry, which Phase 3 should count. +- *The ordinal is `o`, not `i`*, because `i` is the envelope's identifier, which `->` grafts onto an extracted entry. An extracted entry also carries `j`, its ordinal in the document, which its AAD is bound to, and a walk adds `d`, its depth below the start. +- *The subtree walk returns a node's descendants and not the node itself.* The walk addresses nodes by their parent, and the design's wire has no field that names a node by its own token. The client reads the node's kind from its children's labels; a node with no descendants, an empty container or a scalar, yields no rows, and `->` on its selector returns the node. A self tag per node would close this at sixteen bytes an entry and is left to Phase 4. +- *Containment is not `STRICT` in SQL.* PostgreSQL inlines a `STRICT` SQL function only when it can prove the body is NULL for a NULL argument, and the `OR` that adds the needle's `alt` probe is not provably NULL; un-inlined, the call was an opaque filter and the GIN index never engaged. `eql_v4.json_tree_contains` and the `@>` / `<@` wrappers over it are declared without `STRICT` and return NULL for a NULL side on their own. +- *`<@` cannot match a stored root scalar inside a needle root array*, since a stored document carries no `alt` pair. The oracle records the case; `@>` is exact. +- *Value selectors compare strings byte for byte*, with no Unicode normalization, because `jsonb @>` does, and the oracle is agreement with `jsonb @>`. The scalar equality index normalizes to NFC; a JSON value does not. Numbers canonicalize by value, so `2` and `2.0` are one value, as `jsonb` compares them. +- *`AtPath` is the containment of the needle document that holds the value at the path*, with a position in the path read as the wildcard, rather than a single `{vs}` entry: a leaf inside an array element sits in that element's group of the projection, and the needle has to say which group. Containment of that needle is equality only for a scalar, so a container value is refused (`TermError::JsonValueNotScalar`; `op::Contains` asks the containment question), and the needle carries no `alt`, so `("$", 1)` does not match a stored `[1]`. +- *The data-plan form and the catalog are not done.* The Go guest, `stashgen`, the target table and `v4::all()` do not know the family; `IndexSpec::Json` crosses the wire and is refused with a reason. The family is hand-written in `eql-bindings` as the SteVec structs are. Both are the next bindings issue. +- *A document is at most 127 levels deep*, under either root option. The sealed root's plaintext is the document as JSON, and `serde_json` parses JSON back only to 128 levels of nesting, so the 512 levels the walk first admitted sealed documents that then never opened. The bound is the reader's, so a document too deep fails when it is sealed, while the caller still holds it, and not when it is read. +- *A stored document is read as untrusted input.* Its chain fields are not authenticated, and whoever holds a token can write tags that form a cycle or a chain deeper than any sealed document. `JsonDocument::children` checks each entry at its own ordinal instead of looping over a fan-out the document chooses; `JsonDocument::subtree` and the SQL walks stop below 127 levels, and the Rust walk also stops once it has returned as many entries as the document holds; the rebuild of a subtree's rows refuses a repeated row and a nesting deeper than 127 levels as `OpenedType`. Each would otherwise have overflowed the stack, looped, or allocated what the document asked for. +- *Nothing wipes the plaintext copies the walk makes* (the per-node scalars it clones before sealing), where the scalar paths hold theirs in `Zeroizing`. A `serde_json::Value` does not zeroize; a follow-up should walk the caller's value without copying scalars out of it. + ## Decisions to take before Phase 1 1. **Token encoding on the wire:** hex, matching v3's `s`, or base64url, saving about 40 characters per entry. Proposal: measure in Phase 3, default to base64url. diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 2a78111e1..68a877461 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -116,6 +116,26 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. that orders by ADR-0002's order key: floats as PostgreSQL's `float8` sorts them, text by its fold. No `Between` yet: it needs EQL v4's range query domains. +- **`IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, + `DoubleOrdOre`, `TextOrdOre` and `TextSearchOre`**: a field tagged `encrypt_into=` one of + them gets `Lt`, `Lte`, `Gt` and `Gte` (and on text `Eq`, and on + `TextSearchOre` `Matches`), as the `Ord` types do. They store a block ORE + term (ore-rs's 6-bit schemes), compared by EQL v4's comparator; each + write and each query carries a fresh nonce, so two terms of one value + never share bytes. Text whose fold is longer than 1023 bytes or ends in + NUL is refused, stored or queried. The engine reports their ordering + operators through the index `ore_block` (`record.OreBlock`, + `stashgen.IndexOreBlock`), which a struct reaches only by naming one of + these types: it is not a tag word. +- **`TextSearch`**: a `string` field tagged `encrypt_into=TextSearch` gets + `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, one column for all of them. + The stored value keeps all three terms. Text shorter than three + characters, an empty string included, is stored with an empty match + filter, where `TextMatch` refuses it with `ErrTerm`: `Eq`, `Lt`, `Lte`, + `Gt` and `Gte` find the row, and no `Matches` query does. Each query + carries only the term its operator reads, so `Eq` and `Lt` take a + two-letter value and `Matches` refuses one (`ErrTerm`). Needs the EQL v4 + SQL from this release, whose `eql_v4.query_text_search` accepts one term. - **Ordering methods on text.** A string field with `index=ore` or `index=ope` gets `Lt`, `Lte`, `Gt`, `Gte` and `Between`. Text orders by its folded form (ADR-0002: case and accents folded, under Unicode 16), so diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 89e095cae..2603cdf20 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -158,23 +158,24 @@ The engine produces these EQL types today, each from one Go field type: | `encrypt_into` | Go field | |---|---| -| `TextEq`, `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe` | `string` | -| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe` | `int16`, or any Go integer type | -| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe` | `int32`, or any Go integer type | -| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe` | `int64`, or any Go integer type | +| `TextEq`, `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe`, `TextOrdOre`, `TextSearch`, `TextSearchOre` | `string` | +| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | `int16`, or any Go integer type | +| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | `int32`, or any Go integer type | +| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | `int64`, or any Go integer type | | `Boolean` | `bool` | -| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe` | `float32` | -| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe` | `float64` | +| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `float32` | +| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `float64` | | `Date`, `DateEq` | `encrypt.Date` | | `Timestamp`, `TimestampEq` | `time.Time` | | `Numeric`, `NumericEq` | `encrypt.Decimal` | -The `Eq` types answer `Eq`; the `Ord` and `OrdOpe` types answer `Lt`, `Lte`, `Gt` and `Gte`, and on text `Eq` too; `Text`, `Smallint`, `Integer`, `Bigint`, `Boolean`, `Real`, `Double`, `Date`, `Timestamp` and `Numeric` are stored and read only, with no query method. +The `Eq` types answer `Eq`; the `Ord`, `OrdOpe` and `OrdOre` types answer `Lt`, `Lte`, `Gt` and `Gte`, and on text `Eq` too; `TextMatch` answers `Matches`, and `TextSearch` and `TextSearchOre` all of `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, each query carrying only the term its operator reads; `Text`, `Smallint`, `Integer`, `Bigint`, `Boolean`, `Real`, `Double`, `Date`, `Timestamp` and `Numeric` are stored and read only, with no query method. The ordered date, timestamp and numeric types are refused: the engine derives no order term for these kinds yet. An ordered type has no `Between` method yet, because a `BETWEEN` operand needs EQL v4's range query domains. Any built-in Go integer field takes an integer type (ADR-0002) and seals at the type's width: an `int16` field always fits an `Integer`, and an `int64` field is range checked value by value, `Encrypt` refusing a value that does not fit with `encrypt.ErrEncoding`. `encrypt.Int128` and `encrypt.Uint128` are not converted. A float's width must match: a `float64` field is not a `Real`. A `TextMatch` field refuses text shorter than three characters, such as an empty string or a two-letter code, because it has no 3-grams to index; its query refuses such text too. +A `TextSearch` field stores such text with an empty match filter, so its `Eq`, `Lt`, `Lte`, `Gt` and `Gte` queries find it and no `Matches` query does; those queries take such text too, and its `Matches` query refuses it. Put the column on the left of `@@` (`WHERE name @@ $1::eql_v4.query_text_match`): the other operand order finds nothing until #1171 is fixed. `encrypt_into` with any other type is refused with the type's name and the reason it is not produced yet. Separate columns work today for four indexes: `equality`, `match`, `ore` and `ope`. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index a9558ab1a..84ab1ddb9 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -142,7 +142,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { t.Fatalf("the generated file does not name the EQL type:\n%s", written) } // A type the engine cannot produce yet: refused by name, nothing written. - unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=TextOrdOre", 1) + unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=DateOrdOre", 1) dir = writeModule(t, map[string]string{"model.go": unproducible}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { @@ -150,7 +150,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // Refused by the generator with the engine's reason, before the engine // is asked: se_targets lists the type as not producible. - if !strings.Contains(stderr.String(), "cannot produce the EQL type TextOrdOre yet") { + if !strings.Contains(stderr.String(), "cannot produce the EQL type DateOrdOre yet") { t.Fatalf("stderr = %q", stderr.String()) } if _, err := os.Stat(filepath.Join(dir, "user_stash.go")); !os.IsNotExist(err) { diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index e7ff591eb..d6a82d91c 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -487,24 +487,25 @@ The EQL types the engine produces, and the Go field types each takes: |---|---|---| | `TextEq` | `string` | `=`, `<>` | | `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) | -| `TextOrd`, `TextOrdOpe` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `TextOrd`, `TextOrdOpe`, `TextOrdOre` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `TextSearch`, `TextSearchOre` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY`, `@@` | | `Text` | `string` | none: stored and read only | | `Smallint` | `int16`, or any Go integer type (see below) | none | | `SmallintEq` | the same | `=`, `<>` | -| `SmallintOrd`, `SmallintOrdOpe` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Integer` | `int32`, or any Go integer type (see below) | none | | `IntegerEq` | the same | `=`, `<>` | -| `IntegerOrd`, `IntegerOrdOpe` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Bigint` | `int64`, or any Go integer type | none | | `BigintEq` | the same | `=`, `<>` | -| `BigintOrd`, `BigintOrdOpe` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Boolean` | `bool` | none | | `Real` | `float32` | none | | `RealEq` | `float32` | `=`, `<>` | -| `RealOrd`, `RealOrdOpe` | `float32` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `float32` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Double` | `float64` | none | | `DoubleEq` | `float64` | `=`, `<>` | -| `DoubleOrd`, `DoubleOrdOpe` | `float64` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `float64` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Date` | `encrypt.Date` | none | | `DateEq` | `encrypt.Date` | `=`, `<>` | | `Timestamp` | `time.Time` | none | @@ -523,7 +524,13 @@ with `ErrEncoding`, naming the field's kind and not the value. exactly their own width. A `TextMatch` value must be at least three characters long: shorter text has -no 3-grams to index and is refused. The number equality types compare as +no 3-grams to index and is refused. A `TextSearch` value of such text, an +empty string included, is stored with an empty match filter: `Eq`, `Lt`, +`Lte`, `Gt` and `Gte` find it through its other terms, and no `Matches` +query does. A `TextSearch` query carries only the term its operator reads, +so `Eq` and `Lt` on two-letter text work and `Matches` on it is refused; EQL +v4 raises if a query lacks the term its operator compares. The number +equality types compare as PostgreSQL compares the plain types: an integer equals itself whatever its width, and for floats `-0.0 = 0.0` and `NaN = NaN`; the stored value still decrypts to exactly what was written. The same holds for the date, @@ -553,8 +560,19 @@ answers `Eq` through an equality term only. A text order type also stores `TextEq`'s equality term, and `=` reads that, so `Café` and `cafe` are neither `<` nor `>` each other and not `=` either. Text holding a code point Unicode 16 does not assign has no order term and is refused. There is no -`Between` method yet: a `BETWEEN` needs EQL v4's range query domains. The rest of -the catalog (ORE ordering, ordered dates, timestamps and decimals, JSON) is listed in +`Between` method yet: a `BETWEEN` needs EQL v4's range query domains. + +The `OrdOre` types and `TextSearchOre` order by the same key, through a block +ORE term (ore-rs's 6-bit schemes) in place of the OPE term. Each term +carries a random nonce, so EQL v4's own comparator orders them, not their +bytes: sort with `ORDER BY eql_v4.ord_term_ore(col)`, and index with +`CREATE INDEX ON t USING btree (eql_v4.ord_term_ore(col))`. That index needs +the ORE operator class, which only a superuser install creates; elsewhere +the installer disables these domains, and the `Ord` types are the ones to +use. Text whose fold is longer than 1023 bytes, or ends in NUL, has no block +ORE term and is refused, stored or queried. + +The rest of the catalog (ordered dates, timestamps and decimals, JSON) is listed in `eql.Types` with the reason it is not produced yet, and `stashgen` refuses it with that reason. The guest builds the value: the generated code imports `encrypt/eql`, which embeds the build of the engine that holds the diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index a4ce1228d..3ea4c81eb 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -178,7 +178,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { switch o := record.Output(s); o { // "json" is the SteVec document index, which no plan output // carries. - case record.Equality, record.Match, record.Ore, record.Ope, "json": + case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock, "json": t.Indexes = append(t.Indexes, o) default: return t, fmt.Errorf("unknown index %q", s) @@ -294,10 +294,10 @@ func parseFidelity(s string) (record.Fidelity, error) { } // parseOperatorIndex reads the index that answers an operator: eq, match, -// ore or ope. +// ore, ope or ore_block. func parseOperatorIndex(s string) (record.Output, error) { switch o := record.Output(s); o { - case record.Equality, record.Match, record.Ore, record.Ope: + case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock: return o, nil } return "", fmt.Errorf("unknown index %q", s) diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 8d23c8332..b47e285eb 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -11,51 +11,51 @@ import "database/sql/driver" var Types = []Type{ {Name: "Integer", GoName: "Integer", Family: "integer", Suffix: "", Plaintext: "int32", SQLDomain: "public.eql_v4_integer", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "IntegerEq", GoName: "IntegerEq", Family: "integer", Suffix: "Eq", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_eq", Indexes: []string{"eq"}, Query: "IntegerEqQuery", QuerySQLDomain: "eql_v4.query_integer_eq", Producible: true, Reason: ""}, - {Name: "IntegerOrdOre", GoName: "IntegerOrdOre", Family: "integer", Suffix: "OrdOre", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ore", Indexes: []string{"ore"}, Query: "IntegerOrdOreQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "IntegerOrdOre", GoName: "IntegerOrdOre", Family: "integer", Suffix: "OrdOre", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ore", Indexes: []string{"ore_block"}, Query: "IntegerOrdOreQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ore", Producible: true, Reason: ""}, {Name: "IntegerOrd", GoName: "IntegerOrd", Family: "integer", Suffix: "Ord", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord", Indexes: []string{"ope"}, Query: "IntegerOrdQuery", QuerySQLDomain: "eql_v4.query_integer_ord", Producible: true, Reason: ""}, {Name: "IntegerOrdOpe", GoName: "IntegerOrdOpe", Family: "integer", Suffix: "OrdOpe", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ope", Indexes: []string{"ope"}, Query: "IntegerOrdOpeQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ope", Producible: true, Reason: ""}, {Name: "Smallint", GoName: "Smallint", Family: "smallint", Suffix: "", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "SmallintEq", GoName: "SmallintEq", Family: "smallint", Suffix: "Eq", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_eq", Indexes: []string{"eq"}, Query: "SmallintEqQuery", QuerySQLDomain: "eql_v4.query_smallint_eq", Producible: true, Reason: ""}, - {Name: "SmallintOrdOre", GoName: "SmallintOrdOre", Family: "smallint", Suffix: "OrdOre", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ore", Indexes: []string{"ore"}, Query: "SmallintOrdOreQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "SmallintOrdOre", GoName: "SmallintOrdOre", Family: "smallint", Suffix: "OrdOre", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ore", Indexes: []string{"ore_block"}, Query: "SmallintOrdOreQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ore", Producible: true, Reason: ""}, {Name: "SmallintOrd", GoName: "SmallintOrd", Family: "smallint", Suffix: "Ord", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord", Indexes: []string{"ope"}, Query: "SmallintOrdQuery", QuerySQLDomain: "eql_v4.query_smallint_ord", Producible: true, Reason: ""}, {Name: "SmallintOrdOpe", GoName: "SmallintOrdOpe", Family: "smallint", Suffix: "OrdOpe", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ope", Indexes: []string{"ope"}, Query: "SmallintOrdOpeQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ope", Producible: true, Reason: ""}, {Name: "Bigint", GoName: "Bigint", Family: "bigint", Suffix: "", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "BigintEq", GoName: "BigintEq", Family: "bigint", Suffix: "Eq", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_eq", Indexes: []string{"eq"}, Query: "BigintEqQuery", QuerySQLDomain: "eql_v4.query_bigint_eq", Producible: true, Reason: ""}, - {Name: "BigintOrdOre", GoName: "BigintOrdOre", Family: "bigint", Suffix: "OrdOre", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ore", Indexes: []string{"ore"}, Query: "BigintOrdOreQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "BigintOrdOre", GoName: "BigintOrdOre", Family: "bigint", Suffix: "OrdOre", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ore", Indexes: []string{"ore_block"}, Query: "BigintOrdOreQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ore", Producible: true, Reason: ""}, {Name: "BigintOrd", GoName: "BigintOrd", Family: "bigint", Suffix: "Ord", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord", Indexes: []string{"ope"}, Query: "BigintOrdQuery", QuerySQLDomain: "eql_v4.query_bigint_ord", Producible: true, Reason: ""}, {Name: "BigintOrdOpe", GoName: "BigintOrdOpe", Family: "bigint", Suffix: "OrdOpe", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ope", Indexes: []string{"ope"}, Query: "BigintOrdOpeQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ope", Producible: true, Reason: ""}, {Name: "Date", GoName: "Date", Family: "date", Suffix: "", Plaintext: "date", SQLDomain: "public.eql_v4_date", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "DateEq", GoName: "DateEq", Family: "date", Suffix: "Eq", Plaintext: "date", SQLDomain: "public.eql_v4_date_eq", Indexes: []string{"eq"}, Query: "DateEqQuery", QuerySQLDomain: "eql_v4.query_date_eq", Producible: true, Reason: ""}, - {Name: "DateOrdOre", GoName: "DateOrdOre", Family: "date", Suffix: "OrdOre", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ore", Indexes: []string{"ore"}, Query: "DateOrdOreQuery", QuerySQLDomain: "eql_v4.query_date_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "DateOrdOre", GoName: "DateOrdOre", Family: "date", Suffix: "OrdOre", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ore", Indexes: []string{"ore_block"}, Query: "DateOrdOreQuery", QuerySQLDomain: "eql_v4.query_date_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "DateOrd", GoName: "DateOrd", Family: "date", Suffix: "Ord", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord", Indexes: []string{"ope"}, Query: "DateOrdQuery", QuerySQLDomain: "eql_v4.query_date_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "DateOrdOpe", GoName: "DateOrdOpe", Family: "date", Suffix: "OrdOpe", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ope", Indexes: []string{"ope"}, Query: "DateOrdOpeQuery", QuerySQLDomain: "eql_v4.query_date_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "Timestamp", GoName: "Timestamp", Family: "timestamp", Suffix: "", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "TimestampEq", GoName: "TimestampEq", Family: "timestamp", Suffix: "Eq", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_eq", Indexes: []string{"eq"}, Query: "TimestampEqQuery", QuerySQLDomain: "eql_v4.query_timestamp_eq", Producible: true, Reason: ""}, - {Name: "TimestampOrdOre", GoName: "TimestampOrdOre", Family: "timestamp", Suffix: "OrdOre", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ore", Indexes: []string{"ore"}, Query: "TimestampOrdOreQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "TimestampOrdOre", GoName: "TimestampOrdOre", Family: "timestamp", Suffix: "OrdOre", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ore", Indexes: []string{"ore_block"}, Query: "TimestampOrdOreQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "TimestampOrd", GoName: "TimestampOrd", Family: "timestamp", Suffix: "Ord", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord", Indexes: []string{"ope"}, Query: "TimestampOrdQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "TimestampOrdOpe", GoName: "TimestampOrdOpe", Family: "timestamp", Suffix: "OrdOpe", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ope", Indexes: []string{"ope"}, Query: "TimestampOrdOpeQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "Numeric", GoName: "Numeric", Family: "numeric", Suffix: "", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "NumericEq", GoName: "NumericEq", Family: "numeric", Suffix: "Eq", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_eq", Indexes: []string{"eq"}, Query: "NumericEqQuery", QuerySQLDomain: "eql_v4.query_numeric_eq", Producible: true, Reason: ""}, - {Name: "NumericOrdOre", GoName: "NumericOrdOre", Family: "numeric", Suffix: "OrdOre", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ore", Indexes: []string{"ore"}, Query: "NumericOrdOreQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "NumericOrdOre", GoName: "NumericOrdOre", Family: "numeric", Suffix: "OrdOre", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ore", Indexes: []string{"ore_block"}, Query: "NumericOrdOreQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "NumericOrd", GoName: "NumericOrd", Family: "numeric", Suffix: "Ord", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord", Indexes: []string{"ope"}, Query: "NumericOrdQuery", QuerySQLDomain: "eql_v4.query_numeric_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "NumericOrdOpe", GoName: "NumericOrdOpe", Family: "numeric", Suffix: "OrdOpe", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ope", Indexes: []string{"ope"}, Query: "NumericOrdOpeQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "Text", GoName: "Text", Family: "text", Suffix: "", Plaintext: "string", SQLDomain: "public.eql_v4_text", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "TextEq", GoName: "TextEq", Family: "text", Suffix: "Eq", Plaintext: "string", SQLDomain: "public.eql_v4_text_eq", Indexes: []string{"eq"}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true, Reason: ""}, {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v4_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v4.query_text_match", Producible: true, Reason: ""}, - {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ore", Indexes: []string{"eq", "ore"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v4.query_text_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ore", Indexes: []string{"eq", "ore_block"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v4.query_text_ord_ore", Producible: true, Reason: ""}, {Name: "TextOrd", GoName: "TextOrd", Family: "text", Suffix: "Ord", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord", Indexes: []string{"eq", "ope"}, Query: "TextOrdQuery", QuerySQLDomain: "eql_v4.query_text_ord", Producible: true, Reason: ""}, {Name: "TextOrdOpe", GoName: "TextOrdOpe", Family: "text", Suffix: "OrdOpe", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ope", Indexes: []string{"eq", "ope"}, Query: "TextOrdOpeQuery", QuerySQLDomain: "eql_v4.query_text_ord_ope", Producible: true, Reason: ""}, - {Name: "TextSearchOre", GoName: "TextSearchOre", Family: "text", Suffix: "SearchOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_search_ore", Indexes: []string{"eq", "ore", "match"}, Query: "TextSearchOreQuery", QuerySQLDomain: "eql_v4.query_text_search_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, - {Name: "TextSearch", GoName: "TextSearch", Family: "text", Suffix: "Search", Plaintext: "string", SQLDomain: "public.eql_v4_text_search", Indexes: []string{"eq", "ope", "match"}, Query: "TextSearchQuery", QuerySQLDomain: "eql_v4.query_text_search", Producible: false, Reason: "every query of this type carries a match term, which text shorter than one 3-gram cannot make, so `=` or `<` on short text would be refused; how a query that is not a match leaves it out is not decided (#1131)"}, + {Name: "TextSearchOre", GoName: "TextSearchOre", Family: "text", Suffix: "SearchOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_search_ore", Indexes: []string{"eq", "ore_block", "match"}, Query: "TextSearchOreQuery", QuerySQLDomain: "eql_v4.query_text_search_ore", Producible: true, Reason: ""}, + {Name: "TextSearch", GoName: "TextSearch", Family: "text", Suffix: "Search", Plaintext: "string", SQLDomain: "public.eql_v4_text_search", Indexes: []string{"eq", "ope", "match"}, Query: "TextSearchQuery", QuerySQLDomain: "eql_v4.query_text_search", Producible: true, Reason: ""}, {Name: "Boolean", GoName: "Boolean", Family: "boolean", Suffix: "", Plaintext: "bool", SQLDomain: "public.eql_v4_boolean", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "Real", GoName: "Real", Family: "real", Suffix: "", Plaintext: "float32", SQLDomain: "public.eql_v4_real", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "RealEq", GoName: "RealEq", Family: "real", Suffix: "Eq", Plaintext: "float32", SQLDomain: "public.eql_v4_real_eq", Indexes: []string{"eq"}, Query: "RealEqQuery", QuerySQLDomain: "eql_v4.query_real_eq", Producible: true, Reason: ""}, - {Name: "RealOrdOre", GoName: "RealOrdOre", Family: "real", Suffix: "OrdOre", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ore", Indexes: []string{"ore"}, Query: "RealOrdOreQuery", QuerySQLDomain: "eql_v4.query_real_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "RealOrdOre", GoName: "RealOrdOre", Family: "real", Suffix: "OrdOre", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ore", Indexes: []string{"ore_block"}, Query: "RealOrdOreQuery", QuerySQLDomain: "eql_v4.query_real_ord_ore", Producible: true, Reason: ""}, {Name: "RealOrd", GoName: "RealOrd", Family: "real", Suffix: "Ord", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord", Indexes: []string{"ope"}, Query: "RealOrdQuery", QuerySQLDomain: "eql_v4.query_real_ord", Producible: true, Reason: ""}, {Name: "RealOrdOpe", GoName: "RealOrdOpe", Family: "real", Suffix: "OrdOpe", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ope", Indexes: []string{"ope"}, Query: "RealOrdOpeQuery", QuerySQLDomain: "eql_v4.query_real_ord_ope", Producible: true, Reason: ""}, {Name: "Double", GoName: "Double", Family: "double", Suffix: "", Plaintext: "float64", SQLDomain: "public.eql_v4_double", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "DoubleEq", GoName: "DoubleEq", Family: "double", Suffix: "Eq", Plaintext: "float64", SQLDomain: "public.eql_v4_double_eq", Indexes: []string{"eq"}, Query: "DoubleEqQuery", QuerySQLDomain: "eql_v4.query_double_eq", Producible: true, Reason: ""}, - {Name: "DoubleOrdOre", GoName: "DoubleOrdOre", Family: "double", Suffix: "OrdOre", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ore", Indexes: []string{"ore"}, Query: "DoubleOrdOreQuery", QuerySQLDomain: "eql_v4.query_double_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "DoubleOrdOre", GoName: "DoubleOrdOre", Family: "double", Suffix: "OrdOre", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ore", Indexes: []string{"ore_block"}, Query: "DoubleOrdOreQuery", QuerySQLDomain: "eql_v4.query_double_ord_ore", Producible: true, Reason: ""}, {Name: "DoubleOrd", GoName: "DoubleOrd", Family: "double", Suffix: "Ord", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord", Indexes: []string{"ope"}, Query: "DoubleOrdQuery", QuerySQLDomain: "eql_v4.query_double_ord", Producible: true, Reason: ""}, {Name: "DoubleOrdOpe", GoName: "DoubleOrdOpe", Family: "double", Suffix: "OrdOpe", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ope", Indexes: []string{"ope"}, Query: "DoubleOrdOpeQuery", QuerySQLDomain: "eql_v4.query_double_ord_ope", Producible: true, Reason: ""}, {Name: "SteVecDocument", GoName: "SteVecDocument", Family: "json", Suffix: "Search", Plaintext: "", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "SteVecQuery", QuerySQLDomain: "eql_v4.query_json", Producible: false, Reason: "the JSON index is a new operation in the engine (#1060)"}, @@ -133,8 +133,8 @@ func (v *IntegerEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrdOre is the EQL type stored in public.eql_v4_integer_ord_ore: the integer family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// IntegerOrdOre is the EQL type stored in public.eql_v4_integer_ord_ore: the integer family with the ore_block index. +// The engine produces it from an int32. type IntegerOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -345,8 +345,8 @@ func (v *SmallintEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrdOre is the EQL type stored in public.eql_v4_smallint_ord_ore: the smallint family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// SmallintOrdOre is the EQL type stored in public.eql_v4_smallint_ord_ore: the smallint family with the ore_block index. +// The engine produces it from an int16. type SmallintOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -557,8 +557,8 @@ func (v *BigintEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrdOre is the EQL type stored in public.eql_v4_bigint_ord_ore: the bigint family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// BigintOrdOre is the EQL type stored in public.eql_v4_bigint_ord_ore: the bigint family with the ore_block index. +// The engine produces it from an int64. type BigintOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -769,7 +769,7 @@ func (v *DateEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrdOre is the EQL type stored in public.eql_v4_date_ord_ore: the date family with the ore index. +// DateOrdOre is the EQL type stored in public.eql_v4_date_ord_ore: the date family with the ore_block index. // The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type DateOrdOre []byte @@ -981,7 +981,7 @@ func (v *TimestampEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrdOre is the EQL type stored in public.eql_v4_timestamp_ord_ore: the timestamp family with the ore index. +// TimestampOrdOre is the EQL type stored in public.eql_v4_timestamp_ord_ore: the timestamp family with the ore_block index. // The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type TimestampOrdOre []byte @@ -1193,7 +1193,7 @@ func (v *NumericEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrdOre is the EQL type stored in public.eql_v4_numeric_ord_ore: the numeric family with the ore index. +// NumericOrdOre is the EQL type stored in public.eql_v4_numeric_ord_ore: the numeric family with the ore_block index. // The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type NumericOrdOre []byte @@ -1452,8 +1452,8 @@ func (v *TextMatchQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrdOre is the EQL type stored in public.eql_v4_text_ord_ore: the text family with the eq and ore indexes. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// TextOrdOre is the EQL type stored in public.eql_v4_text_ord_ore: the text family with the eq and ore_block indexes. +// The engine produces it from a string. type TextOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1593,8 +1593,8 @@ func (v *TextOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextSearchOre is the EQL type stored in public.eql_v4_text_search_ore: the text family with the eq and ore and match indexes. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// TextSearchOre is the EQL type stored in public.eql_v4_text_search_ore: the text family with the eq and ore_block and match indexes. +// The engine produces it from a string. type TextSearchOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1641,7 +1641,7 @@ func (v *TextSearchOreQuery) UnmarshalJSON(b []byte) error { } // TextSearch is the EQL type stored in public.eql_v4_text_search: the text family with the eq and ope and match indexes. -// The engine cannot produce it yet: every query of this type carries a match term, which text shorter than one 3-gram cannot make, so `=` or `<` on short text would be refused; how a query that is not a match leaves it out is not decided (#1131). +// The engine produces it from a string. type TextSearch []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1782,8 +1782,8 @@ func (v *RealEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrdOre is the EQL type stored in public.eql_v4_real_ord_ore: the real family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// RealOrdOre is the EQL type stored in public.eql_v4_real_ord_ore: the real family with the ore_block index. +// The engine produces it from a float32. type RealOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1994,8 +1994,8 @@ func (v *DoubleEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrdOre is the EQL type stored in public.eql_v4_double_ord_ore: the double family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// DoubleOrdOre is the EQL type stored in public.eql_v4_double_ord_ore: the double family with the ore_block index. +// The engine produces it from a float64. type DoubleOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2206,7 +2206,7 @@ func (v *JSON) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerRangeOreQuery is a range query value over the ore index of the integer family: both bounds of one BETWEEN, +// IntegerRangeOreQuery is a range query value over the ore_block index of the integer family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_integer_range_ore takes in eql_v4.between. It ranges over IntegerOrdOre. type IntegerRangeOreQuery []byte @@ -2254,7 +2254,7 @@ func (v *IntegerRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintRangeOreQuery is a range query value over the ore index of the smallint family: both bounds of one BETWEEN, +// SmallintRangeOreQuery is a range query value over the ore_block index of the smallint family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_smallint_range_ore takes in eql_v4.between. It ranges over SmallintOrdOre. type SmallintRangeOreQuery []byte @@ -2302,7 +2302,7 @@ func (v *SmallintRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintRangeOreQuery is a range query value over the ore index of the bigint family: both bounds of one BETWEEN, +// BigintRangeOreQuery is a range query value over the ore_block index of the bigint family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_bigint_range_ore takes in eql_v4.between. It ranges over BigintOrdOre. type BigintRangeOreQuery []byte @@ -2350,7 +2350,7 @@ func (v *BigintRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateRangeOreQuery is a range query value over the ore index of the date family: both bounds of one BETWEEN, +// DateRangeOreQuery is a range query value over the ore_block index of the date family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_date_range_ore takes in eql_v4.between. It ranges over DateOrdOre. type DateRangeOreQuery []byte @@ -2398,7 +2398,7 @@ func (v *DateRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampRangeOreQuery is a range query value over the ore index of the timestamp family: both bounds of one BETWEEN, +// TimestampRangeOreQuery is a range query value over the ore_block index of the timestamp family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_timestamp_range_ore takes in eql_v4.between. It ranges over TimestampOrdOre. type TimestampRangeOreQuery []byte @@ -2446,7 +2446,7 @@ func (v *TimestampRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericRangeOreQuery is a range query value over the ore index of the numeric family: both bounds of one BETWEEN, +// NumericRangeOreQuery is a range query value over the ore_block index of the numeric family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_numeric_range_ore takes in eql_v4.between. It ranges over NumericOrdOre. type NumericRangeOreQuery []byte @@ -2494,7 +2494,7 @@ func (v *NumericRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextRangeOreQuery is a range query value over the ore index of the text family: both bounds of one BETWEEN, +// TextRangeOreQuery is a range query value over the ore_block index of the text family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_text_range_ore takes in eql_v4.between. It ranges over TextOrdOre, TextSearchOre. type TextRangeOreQuery []byte @@ -2542,7 +2542,7 @@ func (v *TextRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealRangeOreQuery is a range query value over the ore index of the real family: both bounds of one BETWEEN, +// RealRangeOreQuery is a range query value over the ore_block index of the real family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_real_range_ore takes in eql_v4.between. It ranges over RealOrdOre. type RealRangeOreQuery []byte @@ -2590,7 +2590,7 @@ func (v *RealRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleRangeOreQuery is a range query value over the ore index of the double family: both bounds of one BETWEEN, +// DoubleRangeOreQuery is a range query value over the ore_block index of the double family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_double_range_ore takes in eql_v4.between. It ranges over DoubleOrdOre. type DoubleRangeOreQuery []byte diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index 3758e261f..94067930a 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -20,14 +20,14 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { } } want := []string{ - "Integer", "IntegerEq", "IntegerOrd", "IntegerOrdOpe", - "Smallint", "SmallintEq", "SmallintOrd", "SmallintOrdOpe", - "Bigint", "BigintEq", "BigintOrd", "BigintOrdOpe", + "Integer", "IntegerEq", "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", + "Smallint", "SmallintEq", "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", + "Bigint", "BigintEq", "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", "DateEq", "Timestamp", "TimestampEq", "Numeric", "NumericEq", - "Text", "TextEq", "TextMatch", "TextOrd", "TextOrdOpe", + "Text", "TextEq", "TextMatch", "TextOrdOre", "TextOrd", "TextOrdOpe", "TextSearchOre", "TextSearch", "Boolean", - "Real", "RealEq", "RealOrd", "RealOrdOpe", - "Double", "DoubleEq", "DoubleOrd", "DoubleOrdOpe", + "Real", "RealEq", "RealOrdOre", "RealOrd", "RealOrdOpe", + "Double", "DoubleEq", "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", } if len(producible) != len(want) { t.Fatalf("producible = %v, want %v", producible, want) @@ -37,7 +37,7 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { t.Fatalf("producible = %v, want %v", producible, want) } } - for name, plaintext := range map[string]string{"Integer": "int32", "IntegerEq": "int32", "IntegerOrd": "int32", "Smallint": "int16", "SmallintEq": "int16", "SmallintOrdOpe": "int16", "Bigint": "int64", "BigintEq": "int64", "BigintOrdOpe": "int64", "Boolean": "bool", "Real": "float32", "RealEq": "float32", "RealOrd": "float32", "Double": "float64", "DoubleEq": "float64", "DoubleOrdOpe": "float64", "TextOrd": "string", "Date": "date", "DateEq": "date", "Timestamp": "timestamp", "TimestampEq": "timestamp", "Numeric": "decimal", "NumericEq": "decimal"} { + for name, plaintext := range map[string]string{"Integer": "int32", "IntegerEq": "int32", "IntegerOrdOre": "int32", "TextOrdOre": "string", "IntegerOrd": "int32", "Smallint": "int16", "SmallintEq": "int16", "SmallintOrdOre": "int16", "SmallintOrdOpe": "int16", "Bigint": "int64", "BigintEq": "int64", "BigintOrdOpe": "int64", "Boolean": "bool", "Real": "float32", "RealEq": "float32", "RealOrd": "float32", "Double": "float64", "DoubleEq": "float64", "DoubleOrdOpe": "float64", "TextOrd": "string", "Date": "date", "DateEq": "date", "Timestamp": "timestamp", "TimestampEq": "timestamp", "Numeric": "decimal", "NumericEq": "decimal"} { if typ, ok := eql.Lookup(name); !ok || typ.Plaintext != plaintext { t.Errorf("%s: Plaintext = %q, want %q", name, typ.Plaintext, plaintext) } diff --git a/languages/golang/encrypt/eql_test.go b/languages/golang/encrypt/eql_test.go index a1cd3fb19..de73e01bd 100644 --- a/languages/golang/encrypt/eql_test.go +++ b/languages/golang/encrypt/eql_test.go @@ -315,7 +315,7 @@ func TestTextEqIsRefusedByTheBuildWithoutEQLTypes(t *testing.T) { if err := eqlChecker.Check(ctx, plan); err != nil { t.Fatalf("the eql build refuses TextEq: %v", err) } - for _, name := range []string{"TextOrdOre", "Nope"} { + for _, name := range []string{"DateOrdOre", "Nope"} { refused := &record.Plan{Context: []string{"users"}, Fields: []record.Field{{Name: "email", Kind: record.String, Target: name}}} if err := eqlChecker.Check(ctx, refused); !errors.Is(err, encrypt.ErrEncoding) { t.Errorf("%s: Check = %v, want ErrEncoding", name, err) diff --git a/languages/golang/encrypt/eql_v4_text_search_test.go b/languages/golang/encrypt/eql_v4_text_search_test.go new file mode 100644 index 000000000..8f23f64d8 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_text_search_test.go @@ -0,0 +1,159 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "errors" + "sort" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// EQL v4's TextSearch, sealed and queried through generated code over the +// deterministic guest build with the EQL types. A stored value carries all +// three terms, short text with the empty filter; a query carries only the +// one its operator reads (ADR-0002, amended 2026-10-19), so Eq and Lt take a +// two-letter name and Matches refuses one. +// packages/eql/tests/encryption/tests/v4_text_search.rs holds the same rules +// against PostgreSQL. + +// keysAndTerms is an EQL value's sorted top-level keys, and the value of +// each key. +func keysAndTerms(t *testing.T, value []byte) (string, map[string]json.RawMessage) { + t.Helper() + var raw map[string]json.RawMessage + if err := json.Unmarshal(value, &raw); err != nil { + t.Fatalf("the EQL value is not JSON: %v", err) + } + if string(raw["v"]) != "4" { + t.Errorf("v = %s, want 4", raw["v"]) + } + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ","), raw +} + +func TestV4TextSearchStoresEveryTermAndQueriesCarryOne(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + encrypted, err := testusers.EncryptEntry(ctx, cipher, []testusers.Entry{{ID: 1, Name: "Alice"}}) + if err != nil { + t.Fatal(err) + } + keys, stored := keysAndTerms(t, encrypted[0].Name) + if keys != "bf,c,hm,i,op,v" { + t.Errorf("stored keys %s, want bf,c,hm,i,op,v", keys) + } + decrypted, err := testusers.DecryptEntry(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + if decrypted[0].Name != "Alice" { + t.Errorf("opened %q", decrypted[0].Name) + } + + f := testusers.EntryFields.Name + // Each method's query carries its operator's term alone, the stored one. + for _, tc := range []struct { + method string + query func(string) ([]byte, error) + term string + }{ + {"Eq", func(v string) ([]byte, error) { return f.Eq(ctx, cipher, v) }, "hm"}, + {"Lt", func(v string) ([]byte, error) { return f.Lt(ctx, cipher, v) }, "op"}, + {"Gte", func(v string) ([]byte, error) { return f.Gte(ctx, cipher, v) }, "op"}, + } { + q, err := tc.query("Alice") + if err != nil { + t.Fatalf("%s: %v", tc.method, err) + } + keys, terms := keysAndTerms(t, q) + if want := strings.Join(sortedWith("i", "v", tc.term), ","); keys != want { + t.Errorf("%s query keys %s, want %s", tc.method, keys, want) + } + if string(terms[tc.term]) != string(stored[tc.term]) { + t.Errorf("%s query's %s is not the stored one", tc.method, tc.term) + } + // Text too short for a 3-gram still has an equality and an order + // term. + short, err := tc.query("Al") + if err != nil { + t.Fatalf("%s on two-letter text: %v", tc.method, err) + } + if keys, _ := keysAndTerms(t, short); keys != strings.Join(sortedWith("i", "v", tc.term), ",") { + t.Errorf("%s query keys on two-letter text %s", tc.method, keys) + } + } + + m, err := f.Matches(ctx, cipher, "Alice") + if err != nil { + t.Fatal(err) + } + keys, terms := keysAndTerms(t, m.Query()) + if keys != "bf,i,v" { + t.Errorf("Matches query keys %s, want bf,i,v", keys) + } + if string(terms["bf"]) != string(stored["bf"]) { + t.Error("the match query's bf is not the stored filter") + } + if ok, err := m.Verify(ctx, []string{"alice", "Bob"}); err != nil || !ok[0] || ok[1] { + t.Errorf("Verify = %v, %v", ok, err) + } + + // A match query on text too short for a 3-gram is refused, never sent + // as an empty filter. + if q, err := f.Matches(ctx, cipher, "Al"); !errors.Is(err, encrypt.ErrTerm) { + t.Errorf("Matches on two-letter text = %v, %v, want ErrTerm", q, err) + } + // Storing such text, an empty string included, keeps every term, the + // match term as the empty filter (where TextMatch refuses the text): Eq + // and Lt find the row through hm and op, and no Matches query does. + short, err := testusers.EncryptEntry(ctx, cipher, []testusers.Entry{{ID: 2, Name: "Al"}, {ID: 3, Name: ""}}) + if err != nil { + t.Fatalf("storing short text in a TextSearch: %v", err) + } + for i, name := range []string{"Al", ""} { + keys, terms := keysAndTerms(t, short[i].Name) + if keys != "bf,c,hm,i,op,v" { + t.Errorf("stored keys for %q %s, want bf,c,hm,i,op,v", name, keys) + } + if string(terms["bf"]) != "[]" { + t.Errorf("stored filter for %q is %s, want []", name, terms["bf"]) + } + eq, err := f.Eq(ctx, cipher, name) + if err != nil { + t.Fatalf("Eq(%q): %v", name, err) + } + if _, q := keysAndTerms(t, eq); string(q["hm"]) != string(terms["hm"]) { + t.Errorf("Eq(%q)'s hm is not the stored one", name) + } + lt, err := f.Lt(ctx, cipher, name) + if err != nil { + t.Fatalf("Lt(%q): %v", name, err) + } + if _, q := keysAndTerms(t, lt); string(q["op"]) != string(terms["op"]) { + t.Errorf("Lt(%q)'s op is not the stored one", name) + } + } + opened, err := testusers.DecryptEntry(ctx, cipher, short) + if err != nil { + t.Fatal(err) + } + if opened[0].Name != "Al" || opened[1].Name != "" { + t.Errorf("opened %q and %q", opened[0].Name, opened[1].Name) + } +} + +// sortedWith is the given keys, sorted. +func sortedWith(keys ...string) []string { + sort.Strings(keys) + return keys +} diff --git a/languages/golang/encrypt/guest/Cargo.lock b/languages/golang/encrypt/guest/Cargo.lock index a5bb1941d..c715159ff 100644 --- a/languages/golang/encrypt/guest/Cargo.lock +++ b/languages/golang/encrypt/guest/Cargo.lock @@ -21,6 +21,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -218,6 +219,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "bumpalo" version = "3.20.3" @@ -230,6 +237,12 @@ version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -1344,6 +1357,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1354,6 +1381,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1369,6 +1405,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1400,6 +1457,35 @@ dependencies = [ "libc", ] +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.8", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -2079,8 +2165,10 @@ dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", + "sha2 0.11.0", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", @@ -2200,6 +2288,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.119" diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 450e655d3..c30b5430b 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -810,7 +810,7 @@ mod tests { assert_eq!(plan_check(&plan), Ok(()), "the eql build runs TextEq"); // A type the engine cannot produce, and one that does not exist, // are refused at plan_check. - for name in ["TextOrdOre", "Nope"] { + for name in ["DateOrdOre", "Nope"] { let refused = encoded(object(vec![( "email", object(vec![ diff --git a/languages/golang/encrypt/guest/src/status.rs b/languages/golang/encrypt/guest/src/status.rs index 2215ac536..07da13b1a 100644 --- a/languages/golang/encrypt/guest/src/status.rs +++ b/languages/golang/encrypt/guest/src/status.rs @@ -432,8 +432,8 @@ mod tests { name: "Nope".into(), }, TargetError::Unproducible { - name: "TextOrdOre".into(), - reason: "block ORE".into(), + name: "DateOrdOre".into(), + reason: "no kind for this family".into(), }, TargetError::Extended { name: "email".into(), diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index b863f0669..816cfce0e 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -79,11 +79,13 @@ impl TargetResolver for EqlTargets { fn query<'a, K: 'static>( &self, name: &str, + index: &stack_encrypt::target::IndexSpec, keyset: &'a stack_encrypt::KeysetCipher<'_, K>, label: &stack_encrypt::Label, plaintext: stack_encrypt::dynamic::ValueFrame, ) -> Result, K>, TargetError> { - eql_bindings::encryption::targets::query(name, keyset, label, plaintext).map_err(convert) + eql_bindings::encryption::targets::query(name, index, keyset, label, plaintext) + .map_err(convert) } } @@ -191,14 +193,17 @@ mod tests { [ "Integer", "IntegerEq", + "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", "Smallint", "SmallintEq", + "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", "Bigint", "BigintEq", + "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", @@ -210,22 +215,27 @@ mod tests { "Text", "TextEq", "TextMatch", + "TextOrdOre", "TextOrd", "TextOrdOpe", + "TextSearchOre", + "TextSearch", "Boolean", "Real", "RealEq", + "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", + "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe" ], "the engine's producible types, in catalog order" ); assert!(matches!( - resolver().resolve("TextOrdOre"), + resolver().resolve("DateOrdOre"), Err(TargetError::Unproducible { .. }) )); assert!(matches!( @@ -252,6 +262,16 @@ mod tests { }), TargetError::Plaintext { target, expected: Some(stack_encrypt::dynamic::ValueKind::String), .. } if target == "TextEq" )); + // The engine asks a type's query only through an index the type + // carries, so one it does not is the engine's failure, not the + // caller's input. + assert!(matches!( + convert(Eql::NoIndex { + name: "TextEq", + index: "ope", + }), + TargetError::Other(_) + )); } /// serde_json quotes the value it refused, and a stored EQL value holds diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 4da201ab8..c386c6e37 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1365,14 +1365,17 @@ fn targets_lists_what_the_build_holds() { [ "Integer", "IntegerEq", + "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", "Smallint", "SmallintEq", + "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", "Bigint", "BigintEq", + "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", @@ -1384,15 +1387,20 @@ fn targets_lists_what_the_build_holds() { "Text", "TextEq", "TextMatch", + "TextOrdOre", "TextOrd", "TextOrdOpe", + "TextSearchOre", + "TextSearch", "Boolean", "Real", "RealEq", + "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", + "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe" ], @@ -1788,13 +1796,20 @@ fn the_catalog_lists_each_types_operators() { ("index", index), ] }; - let orderings = || ["lt", "lte", "gt", "gte"].map(|op| exact(op, "ope")); + // OPE for Ord and OrdOpe, block ORE for OrdOre and SearchOre. + let order_index = if name.ends_with("Ore") { + "ore_block" + } else { + "ope" + }; + let orderings = || ["lt", "lte", "gt", "gte"].map(|op| exact(op, order_index)); match name.as_str() { - "IntegerOrd" | "IntegerOrdOpe" | "SmallintOrd" | "SmallintOrdOpe" | "BigintOrd" - | "BigintOrdOpe" | "RealOrd" | "RealOrdOpe" | "DoubleOrd" | "DoubleOrdOpe" => { + "IntegerOrd" | "IntegerOrdOpe" | "IntegerOrdOre" | "SmallintOrd" | "SmallintOrdOpe" + | "SmallintOrdOre" | "BigintOrd" | "BigintOrdOpe" | "BigintOrdOre" | "RealOrd" + | "RealOrdOpe" | "RealOrdOre" | "DoubleOrd" | "DoubleOrdOpe" | "DoubleOrdOre" => { assert_eq!(listed, orderings().to_vec(), "{name}"); } - "TextOrd" | "TextOrdOpe" => { + "TextOrd" | "TextOrdOpe" | "TextOrdOre" => { let mut want = vec![exact("eq", "eq")]; want.extend(orderings()); assert_eq!(listed, want, "{name}"); @@ -1812,6 +1827,19 @@ fn the_catalog_lists_each_types_operators() { ("arity", "one"), ("index", "match"), ]), + // Each operator through its own index, whose term alone the + // query carries. + "TextSearch" | "TextSearchOre" => { + let mut want = vec![exact("eq", "eq")]; + want.extend(orderings()); + want.push(vec![ + ("operator", "matches"), + ("fidelity", "probabilistic"), + ("arity", "one"), + ("index", "match"), + ]); + assert_eq!(listed, want, "{name}"); + } _ => { // A type the engine does not produce answers nothing, and // the other produced types are storage-only: no twin. diff --git a/languages/golang/encrypt/internal/testusers/directory.go b/languages/golang/encrypt/internal/testusers/directory.go new file mode 100644 index 000000000..7850d7197 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/directory.go @@ -0,0 +1,13 @@ +package testusers + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Entry -name Entry + +// Entry stores a name as EQL v4's TextSearch: one column that answers `=`, +// the orderings and `@@`. Its query carries only the term its operator +// reads, so `=` and `<` work on a two-letter name while `@@` on one is +// refused (ADR-0002, amended 2026-10-19). +type Entry struct { + _ struct{} `stash:"context=directory"` + ID int64 `stash:"id,passthrough"` + Name string `stash:"name,encrypt_into=TextSearch"` +} diff --git a/languages/golang/encrypt/internal/testusers/entry_stash.go b/languages/golang/encrypt/internal/testusers/entry_stash.go new file mode 100644 index 000000000..de9ce0098 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/entry_stash.go @@ -0,0 +1,141 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Entry prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedEntry struct { + ID int64 + Name eql.TextSearch +} + +func (e EncryptedEntry) String() string { + return gensupport.Redacted("EncryptedEntry", map[string]any{"ID": e.ID}, "Name") +} + +func (e EncryptedEntry) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Name") +} + +// Stops compiling when Entry gains, loses, reorders or retypes a field. +var _ = entryShape(Entry{}) + +type entryShape struct { + _ struct{} + ID int64 + Name string +} + +var entryDeclaration = gensupport.Declare("directory"). + Passthrough("id"). + EncryptInto("name", gensupport.String, "TextSearch") + +var entryCodec = gensupport.New(gensupport.Generated[Entry, EncryptedEntry]{ + TypeName: "Entry", + Declaration: entryDeclaration, + PrintsPlaintext: true, + Source: func(v Entry) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "name": v.Name, + } + }, + Seal: func(rec gensupport.Record) (EncryptedEntry, error) { + var e EncryptedEntry + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedEntry{}, err + } + e.Name = eql.TextSearch(rec["name"].EQL) + return e, nil + }, + Open: func(e EncryptedEntry) gensupport.Record { + return gensupport.Record{ + "id": {Value: e.ID}, + "name": {EQL: e.Name}, + } + }, + Value: func(e EncryptedEntry, vals gensupport.Values) (Entry, error) { + var v Entry + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Entry{}, err + } + if v.Name, err = gensupport.Get[string](vals, "name"); err != nil { + return Entry{}, err + } + return v, nil + }, +}) + +// EncryptEntry seals each Entry, with one ZeroKMS request for each 500 sealed +// values, plus one the first time a keyset is used. The result has one element +// for each input, in the same order. +func EncryptEntry(ctx context.Context, cipher *encrypt.Cipher, values []Entry) ([]EncryptedEntry, error) { + return entryCodec.Encrypt(ctx, cipher, values) +} + +// DecryptEntry opens each EncryptedEntry, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. +func DecryptEntry(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedEntry) ([]Entry, error) { + return entryCodec.Decrypt(ctx, d, encrypted) +} + +var EntryFields = struct { + Name EntryNameField +}{ + Name: EntryNameField{gensupport.NewField[string](entryDeclaration, "name")}, +} + +type EntryNameField struct { + field gensupport.Field[string] +} + +func (f EntryNameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearch, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.TextSearch(out.EQL), err +} + +func (f EntryNameField) Eq(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Lt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lt", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Lte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lte", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Gt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gt", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Gte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gte", v) + return eql.TextSearchQuery(term), err +} + +// Matches derives the matches query for v. Its answer can hold rows that do not +// match: decrypt the rows it fetches and keep those Verify reports true. +func (f EntryNameField) Matches(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.EQLMatchQuery[eql.TextSearchQuery, string], error) { + return gensupport.EQLMatches[eql.TextSearchQuery](ctx, f.field, c, v) +} diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index 7b948450a..92559a6f5 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -382,10 +382,10 @@ func TestVerifyAndTargetLimitNeedNoCipher(t *testing.T) { } // The catalog reports, per EQL type, the operators the engine answers: -// TextEq answers eq, exactly, through its equality index, and an OPE-ordered -// type the four orderings of one value through its ope index (Between takes -// a pair, which a type's one query value does not carry), and eq too when it -// also stores an equality term. +// TextEq answers eq, exactly, through its equality index, and an ordered type +// the four orderings of one value through its order index, ope or ore_block +// (Between takes a pair, which a type's one query value does not carry), and +// eq too when it also stores an equality term. func TestTheCatalogListsEachTypesOperators(t *testing.T) { wasm, err := encrypt.EmbeddedGuest() if errors.Is(err, encrypt.ErrGuestNotBuilt) { @@ -407,21 +407,39 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if len(targets) == 0 { t.Skip("this build holds no EQL types: import encrypt/eql") } + families := []string{"Integer", "Smallint", "Bigint", "Real", "Double", "Text"} + // Stored and compared for equality only today: the engine derives no + // order term for these kinds, so their ordered types are refused. + equalityOnly := []string{"Date", "Timestamp", "Numeric"} + // The index a type answers the orderings through: OPE for Ord and + // OrdOpe, block ORE for OrdOre. + orderIndex := map[string]record.Output{} + for _, family := range families { + orderIndex[family+"Ord"] = record.Ope + orderIndex[family+"OrdOpe"] = record.Ope + orderIndex[family+"OrdOre"] = record.OreBlock + } + orderings := func(index record.Output) []record.Operator { + var ops []record.Operator + for _, name := range []string{"lt", "lte", "gt", "gte"} { + ops = append(ops, record.Operator{Name: name, Fidelity: record.Exact, Index: index}) + } + return ops + } for _, target := range targets { + index, ordered := orderIndex[target.Name] switch { - case slices.Contains([]string{"TextEq", "IntegerEq", "SmallintEq", "BigintEq", "RealEq", "DoubleEq", "DateEq", "TimestampEq", "NumericEq"}, target.Name): + case strings.HasSuffix(target.Name, "Eq") && slices.Contains(slices.Concat(families, equalityOnly), strings.TrimSuffix(target.Name, "Eq")): want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} if len(target.Operators) != 1 || target.Operators[0] != want[0] { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } - case slices.Contains([]string{"IntegerOrd", "IntegerOrdOpe", "SmallintOrd", "SmallintOrdOpe", "BigintOrd", "BigintOrdOpe", "RealOrd", "RealOrdOpe", "DoubleOrd", "DoubleOrdOpe", "TextOrd", "TextOrdOpe"}, target.Name): + case ordered: var want []record.Operator if strings.HasPrefix(target.Name, "Text") { want = append(want, record.Operator{Name: "eq", Fidelity: record.Exact, Index: record.Equality}) } - for _, name := range []string{"lt", "lte", "gt", "gte"} { - want = append(want, record.Operator{Name: name, Fidelity: record.Exact, Index: record.Ope}) - } + want = append(want, orderings(index)...) if !slices.Equal(target.Operators, want) { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } @@ -430,6 +448,18 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if len(target.Operators) != 1 || target.Operators[0] != want { t.Fatalf("TextMatch answers %+v, want %+v", target.Operators, want) } + case target.Name == "TextSearch" || target.Name == "TextSearchOre": + // Each through its own index: the query carries that term alone. + index := record.Ope + if target.Name == "TextSearchOre" { + index = record.OreBlock + } + want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} + want = append(want, orderings(index)...) + want = append(want, record.Operator{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match}) + if !slices.Equal(target.Operators, want) { + t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) + } case len(target.Operators) != 0: // Every other type is storage-only or not produced. t.Fatalf("%s (producible %v) answers %+v, want none", target.Name, target.Producible, target.Operators) diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 34b398af5..448dfdbfc 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -6,8 +6,8 @@ // to check one. // // The spellings here are wire format, fixed by stack-encrypt's -// dynamic::record: the output keys "c", "eq", "match", "ore", "ope" and -// "passthrough", and the type names "bool", "int8", "int16", "int32", +// dynamic::record: the output keys "c", "eq", "match", "ore", "ope", +// "ore_block" and "passthrough", and the type names "bool", "int8", "int16", "int32", // "int64", "int128", "uint8", "uint16", "uint32", "uint64", "uint128", // "float32", "float64", "decimal", "date", "timestamp", "string" and // "bytes". A field's context is @@ -75,6 +75,7 @@ const ( Match Output = "match" Ore Output = "ore" Ope Output = "ope" + OreBlock Output = "ore_block" Passthrough Output = "passthrough" ) @@ -428,7 +429,7 @@ type Sealed = map[string]Outputs // Target is one EQL type as se_targets lists it. The entry's keys are wire // format, written by eql-bindings' serialiser and read by [ParseTargets]: // name, family, suffix, plaintext (a ValueKind name, or null), sql_domain, -// indexes (eq | match | ore | ope | json), query (or null), query_sql_domain +// indexes (eq | match | ore | ope | ore_block | json), query (or null), query_sql_domain // (or null), producible, reason (or null), and operators, which the engine // adds: the operators it answers for the type, each an object of operator, // fidelity (exact | probabilistic), arity (one | pair) and index. diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index eff71aa49..273a7c978 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -154,6 +154,11 @@ const ( IndexOre IndexName = "ore" IndexOpe IndexName = "ope" IndexJSON IndexName = "json" + // IndexOreBlock is block ORE (ore-rs's 6-bit schemes): the index of an + // EQL type's ORE term (`ob`), TextOrdOre's and IntegerOrdOre's. It is + // not a tag word: a field reaches it by naming such an EQL type + // (encrypt_into=), and indexNames leaves it out. + IndexOreBlock IndexName = "ore_block" ) // indexNames lists every index, in the order the generated type lists their @@ -172,6 +177,8 @@ func (n IndexName) GoName() string { return "Ore" case IndexOpe: return "Ope" + case IndexOreBlock: + return "OreBlock" case IndexJSON: return "JSON" } diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index d8b71bca7..50255cf33 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -277,8 +277,11 @@ var outputOfIndex = map[IndexName]record.Output{ IndexEquality: record.Equality, IndexMatch: record.Match, IndexOre: record.Ore, IndexOpe: record.Ope, } +// indexOfOutput also names block ORE, which only an EQL type carries: it has +// no entry in outputOfIndex, so no tag asks for it. var indexOfOutput = map[record.Output]IndexName{ record.Equality: IndexEquality, record.Match: IndexMatch, record.Ore: IndexOre, record.Ope: IndexOpe, + record.OreBlock: IndexOreBlock, } // wireKind is the record kind of a Go type: the same mapping the emitter diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 7aeba26bc..61be14f85 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -33,6 +33,10 @@ var eqlTypes = []struct { {stashgen.EQLType{Name: "Text", Plaintext: stashgen.KindString, Producible: true}, ""}, {stashgen.EQLType{Name: "TextEq", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality}, Query: "TextEqQuery", Producible: true}, ""}, {stashgen.EQLType{Name: "TextMatch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexMatch}, Query: "TextMatchQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextSearch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality, stashgen.IndexOpe, stashgen.IndexMatch}, Query: "TextSearchQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextOrdOre", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality, stashgen.IndexOreBlock}, Query: "TextOrdOreQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextSearchOre", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality, stashgen.IndexOreBlock, stashgen.IndexMatch}, Query: "TextSearchOreQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "IntegerOrdOre", Plaintext: stashgen.KindInt, Indexes: []stashgen.IndexName{stashgen.IndexOreBlock}, Query: "IntegerOrdOreQuery", Producible: true}, "int32"}, {stashgen.EQLType{Name: "Boolean", Plaintext: stashgen.KindBool, Producible: true}, ""}, {stashgen.EQLType{Name: "Real", Plaintext: stashgen.KindFloat, Producible: true}, "float32"}, {stashgen.EQLType{Name: "Double", Plaintext: stashgen.KindFloat, Producible: true}, "float64"}, @@ -182,7 +186,7 @@ func (e Static) operators(ctx context.Context, d stashgen.Declaration) (map[stri // answeredBy mirrors OpSpec::answered_by in stack-encrypt: eq through an // equality index, matches through a match index, and the orderings through -// ORE, or OPE when there is no ORE, over an integer or text; each only +// ORE, else block ORE, else OPE, over an integer or text; each only // where the index applies to the type. func answeredBy(operator string, t stashgen.GoType, indexes []stashgen.IndexName) (stashgen.IndexName, bool) { declared := func(want stashgen.IndexName) bool { @@ -203,8 +207,10 @@ func answeredBy(operator string, t stashgen.GoType, indexes []stashgen.IndexName if !orderable { return "", false } - if declared(stashgen.IndexOre) { - return stashgen.IndexOre, true + for _, index := range []stashgen.IndexName{stashgen.IndexOre, stashgen.IndexOreBlock} { + if declared(index) { + return index, true + } } return stashgen.IndexOpe, declared(stashgen.IndexOpe) } @@ -252,7 +258,7 @@ func indexApplies(name stashgen.IndexName, t stashgen.GoType) error { return nil } return fmt.Errorf("match applies to a string, not to %s", t) - case stashgen.IndexOre, stashgen.IndexOpe: + case stashgen.IndexOre, stashgen.IndexOpe, stashgen.IndexOreBlock: switch t.Kind { case stashgen.KindString, stashgen.KindBytes, stashgen.KindFloat: return nil diff --git a/languages/golang/stashgen/golden_test.go b/languages/golang/stashgen/golden_test.go index fd66a8b48..f721f299a 100644 --- a/languages/golang/stashgen/golden_test.go +++ b/languages/golang/stashgen/golden_test.go @@ -39,6 +39,9 @@ var goldenCases = map[string]goldenCase{ // the context field to a column. "tenants": {req: stashgen.Request{Type: "Note", Models: []stashgen.ModelRequest{{Name: "Rows", Type: "NoteRow"}}}, golden: "note_stash.go.golden"}, "foreign": {req: stashgen.Request{Type: "individualStash", For: "pb.Individual"}, golden: "individualstash_stash.go.golden"}, + // An EQL type with three indexes, TextSearch: Eq, the orderings and + // Matches, each returning its own query. + "directory": {req: stashgen.Request{Type: "Entry"}, golden: "entry_stash.go.golden"}, } func TestGolden(t *testing.T) { diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index d91f5de3e..027e45a02 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -93,7 +93,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with an interface", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAny any"), stashgen.Request{Type: "User"}, "Any", "is not a type JSON carries"}, {"an opaque struct with a struct JSON would truncate", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx hidden") + "\ntype hidden struct {\n\tA int\n\tb int\n}\n", stashgen.Request{Type: "User"}, "Tx", "JSON would drop"}, {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, - {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int32 `stash:\"age,encrypt_into=IntegerOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type IntegerOrdOre"}, + {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, {"the json index, not in the engine yet", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "cannot derive the json index yet"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, diff --git a/languages/golang/stashgen/testdata/cases/directory/directory.go b/languages/golang/stashgen/testdata/cases/directory/directory.go new file mode 100644 index 000000000..23b1f8918 --- /dev/null +++ b/languages/golang/stashgen/testdata/cases/directory/directory.go @@ -0,0 +1,12 @@ +package directory + +//go:generate go tool stashgen -type Entry + +// Entry stores a name as EQL v4's TextSearch: one column that answers +// =, the orderings and @@, each query carrying only the term its operator +// reads. +type Entry struct { + _ struct{} `stash:"context=directory"` + ID int64 `stash:"id,passthrough" db:"id"` + Name string `stash:"name,encrypt_into=TextSearch" db:"name"` +} diff --git a/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden b/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden new file mode 100644 index 000000000..d0393bf7c --- /dev/null +++ b/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden @@ -0,0 +1,141 @@ +// Code generated by stashgen. DO NOT EDIT. + +package directory + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Entry prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedEntry struct { + ID int64 `db:"id"` + Name eql.TextSearch `db:"name"` +} + +func (e EncryptedEntry) String() string { + return gensupport.Redacted("EncryptedEntry", map[string]any{"ID": e.ID}, "Name") +} + +func (e EncryptedEntry) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Name") +} + +// Stops compiling when Entry gains, loses, reorders or retypes a field. +var _ = entryShape(Entry{}) + +type entryShape struct { + _ struct{} + ID int64 + Name string +} + +var declaration = gensupport.Declare("directory"). + Passthrough("id"). + EncryptInto("name", gensupport.String, "TextSearch") + +var codec = gensupport.New(gensupport.Generated[Entry, EncryptedEntry]{ + TypeName: "Entry", + Declaration: declaration, + PrintsPlaintext: true, + Source: func(v Entry) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "name": v.Name, + } + }, + Seal: func(rec gensupport.Record) (EncryptedEntry, error) { + var e EncryptedEntry + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedEntry{}, err + } + e.Name = eql.TextSearch(rec["name"].EQL) + return e, nil + }, + Open: func(e EncryptedEntry) gensupport.Record { + return gensupport.Record{ + "id": {Value: e.ID}, + "name": {EQL: e.Name}, + } + }, + Value: func(e EncryptedEntry, vals gensupport.Values) (Entry, error) { + var v Entry + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Entry{}, err + } + if v.Name, err = gensupport.Get[string](vals, "name"); err != nil { + return Entry{}, err + } + return v, nil + }, +}) + +// Encrypt seals each Entry, with one ZeroKMS request for each 500 sealed +// values, plus one the first time a keyset is used. The result has one element +// for each input, in the same order. +func Encrypt(ctx context.Context, cipher *encrypt.Cipher, directory []Entry) ([]EncryptedEntry, error) { + return codec.Encrypt(ctx, cipher, directory) +} + +// Decrypt opens each EncryptedEntry, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. +func Decrypt(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedEntry) ([]Entry, error) { + return codec.Decrypt(ctx, d, encrypted) +} + +var Fields = struct { + Name NameField +}{ + Name: NameField{gensupport.NewField[string](declaration, "name")}, +} + +type NameField struct { + field gensupport.Field[string] +} + +func (f NameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearch, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.TextSearch(out.EQL), err +} + +func (f NameField) Eq(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Lt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lt", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Lte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lte", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Gt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gt", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Gte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gte", v) + return eql.TextSearchQuery(term), err +} + +// Matches derives the matches query for v. Its answer can hold rows that do not +// match: decrypt the rows it fetches and keep those Verify reports true. +func (f NameField) Matches(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.EQLMatchQuery[eql.TextSearchQuery, string], error) { + return gensupport.EQLMatches[eql.TextSearchQuery](ctx, f.field, c, v) +} diff --git a/languages/golang/stashgen/testdata/cases/directory/go.mod b/languages/golang/stashgen/testdata/cases/directory/go.mod new file mode 100644 index 000000000..c489c9950 --- /dev/null +++ b/languages/golang/stashgen/testdata/cases/directory/go.mod @@ -0,0 +1,7 @@ +module example.com/app + +go 1.26 + +require github.com/cipherstash/stack/languages/golang v0.0.0 + +replace github.com/cipherstash/stack/languages/golang => ../../stubsdk diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go index 17fac8ef2..3bc6a8f87 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go @@ -13,3 +13,9 @@ type TextMatch []byte // TextMatchQuery is the query value for TextMatch. type TextMatchQuery []byte + +// TextSearch is an EQL value with equality, order and match terms. +type TextSearch []byte + +// TextSearchQuery is the query value for TextSearch. +type TextSearchQuery []byte diff --git a/packages/eql/AGENTS.md b/packages/eql/AGENTS.md index a4477f0df..6477cdded 100644 --- a/packages/eql/AGENTS.md +++ b/packages/eql/AGENTS.md @@ -54,11 +54,13 @@ The SQL source never names its schemas or its payload version literally. It writ `{{prefix}}` is the schema and the start of every other name: `{{prefix}}.eq_term`, `{{prefix}}_internal.hmac_256`, `public.{{prefix}}_text_eq`. `{{eql_version}}` appears in every domain CHECK: `VALUE->>'v' = '{{eql_version}}'`. This covers every file under `src/v3/` (hand-written and generated), `src/v3/version.template`, `tasks/pin_search_path_v3.sql` and `tasks/uninstall-v3.sql`. The generator emits the placeholders too: `SCHEMA`, `INTERNAL_SCHEMA`, `PUBLIC_TYPNAME_PREFIX` and `EQL_VERSION` in `crates/eql-codegen/src/consts.rs` are placeholder strings. The Rust bindings are the exception. They describe v3 payloads and keep `eql_domains::PUBLIC_TYPNAME_PREFIX`, the literal `eql_v3_`. +Where the two versions' SQL must differ, the source holds both forms in **version blocks**: lines between `-- {{#v3}}` and `-- {{/v3}}` render only for `eql_v3`, lines between `-- {{#v4}}` and `-- {{/v4}}` only for `eql_v4`, and the directive lines themselves render for neither, so a v4-only block leaves the v3 bundle byte for byte as it was. A directive is a whole line; blocks do not nest. The generator emits them through the `v3_only`/`v3_end`/`v4_only`/`v4_end` template globals. Today they hold the v4 search query operands' optional terms (`query_text_search`, `query_text_search_ore`: at least one of their terms, and each extractor raising through `{{prefix}}_internal.require_query_term`, which lives in a v4 block of `src/v3/scalars/functions.sql`). Diff `release/cipherstash-encrypt.sql` before and after any block you add. + `tasks/build.sh` renders the ordered source into `build/render/eql_v3/` and `build/render/eql_v4/` (gitignored, same relative paths) with `eql-codegen render`, runs the symbol-order and installer-completeness gates on the rendered files, and assembles `release/` from them. The renderer refuses three things: an unknown placeholder, an unterminated `{{`, and a **literal `eql_v3` or `eql_v4`** anywhere in the source, comments included. A literal name renders identically for both versions, so the v4 bundle would reach into v3. Write the placeholder in prose too. #### Files confined to one EQL version -A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Today only the generated range surface does: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7), and are `-- ONLY EQL VERSION: 4`. EQL v3 is what cipherstash-client writes, nothing there produces a range, and its SQL must not change. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the v4-only files; the per-version lists are `build/render/eql_v/deps-ordered.txt`. +A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Two surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `blockers.sql`): `public.eql_v4_json_tree`, `public.eql_v4_json_tree_entry`, `eql_v4.query_json_tree` and `eql_v4.query_json_tree_leaf`, the document Stack Encrypt's JSON index writes, a new family beside `public.eql_v4_json` and not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; the family is not in the catalog yet). Both are `-- ONLY EQL VERSION: 4`. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the v4-only files; the per-version lists are `build/render/eql_v/deps-ordered.txt`. Anything that needs the source as real SQL reads the rendered tree, not `src/`: `docs:validate:documented-sql` runs psql over `build/render/eql_v3/src`, and the Doxygen input filter (`tasks/docs/doxygen-filter.sh`) reads each file's rendered copy from `build/render/eql_v3/`. A new tool that parses `src/v3` must do the same, and depend on `build`; do not substitute the placeholders a second way. diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index e85b3af88..0b689259b 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -582,8 +582,8 @@ dependencies = [ "log", "miette", "opaque-debug", - "orderable-bytes", - "ore-rs", + "orderable-bytes 0.1.1", + "ore-rs 0.8.3", "percent-encoding", "rand 0.8.6", "recipher 0.2.3", @@ -650,7 +650,7 @@ dependencies = [ "blake3", "chrono", "hex", - "orderable-bytes", + "orderable-bytes 0.1.1", "rust_decimal", "subtle", "thiserror 1.0.69", @@ -2667,6 +2667,12 @@ dependencies = [ "rust_decimal", ] +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + [[package]] name = "ore-rs" version = "0.8.3" @@ -2680,7 +2686,7 @@ dependencies = [ "hex", "lazy_static", "num", - "orderable-bytes", + "orderable-bytes 0.1.1", "rand 0.8.6", "rand_chacha 0.3.1", "rust_decimal", @@ -2689,6 +2695,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes 0.2.1", + "rand 0.8.6", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "outref" version = "0.5.2" @@ -4239,8 +4268,10 @@ dependencies = [ "base64ct", "cllw-ore 0.5.2-rc.1", "miette", + "ore-rs 0.9.0", "serde", "serde_json", + "sha2 0.11.0", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index a09be9709..e1a655afe 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -22,6 +22,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `eql_v4.query__range_` domains in the v4 SQL bundle. No range type derives Stack Encrypt's traits yet, because the ordered types they range over are not producible. There are no v3 range types. +- **Rust queries an EQL v4 field through Stack Encrypt's typed query chain** + (`stack-encrypt` feature, #1238). `cipher.query(op::Eq, v).on(&plan)` on a + plan for `TextEq`, `SmallintEq`, `IntegerEq`, `BigintEq`, `RealEq`, + `DoubleEq`, `DateEq`, `TimestampEq` or `NumericEq` + (`Plan::value::().context("table/column").encrypt_into::()`) + returns the type's query twin (`TextEqQuery`, …), and + `cipher.query(op::Matches, text).on(&plan)` on a plan for `TextMatch` + returns a `MatchTwin`: the twin, which it serializes as, + with `verify` and `target_limit` under `EqlMatch`. The query is the bytes + the data path (`stack_encrypt::dynamic::query_term`) derives for the same + field, and `verify` and `target_limit` answer as the data path's do. + eql-codegen writes the `op::Answers` impls into `v4::answers`, which needs + no `dynamic` layer and so not the `targets` feature, from each type's + indexes, by the rule the data path answers an EQL type's operators + with. A producible type whose twin carries an order term (`IntegerOrd`, + `TextOrd`, …) answers nothing in the typed chain yet: the chain sends one + index's term as the twin, and those twins need an order term, or several + terms at once. The encryption tests hold that gap to exactly the ordered + types, so query them through the data path for now. An operator a type + does not answer, or another index named with `.via(..)`, does not + compile. Each answering type is an + `op::LabelledTarget`. The label must name the column, table then column, + and the query must not be extended, or the awaited query is refused + before any keyset is resolved, with a `QueryLabelError` in + `Error::Other`. `eql_bindings::encryption::query` holds `TwinIndex` + (Stack Encrypt's index, its query sent as the twin, for the twin's own + plaintext only), `MatchTwin`, `QueryTwin` and `QueryLabelError`. + `Hmac256` and `BloomFilter` now declare their indexes (`EncryptFrom::indexes`), + so a type holding one declares the equality or match index, and `EqlMatch` + implements `op::MatchCase` (case-insensitive). - **`eql_bindings::v4` needs the `stack-encrypt` feature.** Only Stack Encrypt produces an EQL v4 value (ADR-0002), so the module is compiled with that feature alone, and its producible types derive `EncryptFrom` and @@ -71,6 +101,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 orderable plaintext. Their query twins carry one bound, so the engine answers `<`, `<=`, `>` and `>=` (and `=` on text) through them, not `BETWEEN`. +- **`TextSearch` is producible** (`stack-encrypt` feature), from `String`. + The stored value carries `TextEq`'s `hm`, `TextOrd`'s `op` and + `TextMatch`'s `bf`. Text too short for a 3-gram (an empty string, a + two-letter code) is stored with the empty filter, `"bf": []`, where + `TextMatch` refuses it: `=` and `<` find the row through `hm` and `op`, + and no `@@` query does (a `@@` between two stored values is not a query, + and two stored empty filters match each other there, as in v3). Its `bf` + field is the new + `v4::terms::SearchBloomFilter` (as is `TextSearchOre`'s), whose JSON and + JSON Schema are `BloomFilter`'s and which takes the empty filter; + `TextMatch` and every query keep `BloomFilter`, which refuses it. Its + query is built per index, through + `TextSearchEqualityQuery`, `TextSearchOrderQuery` or + `TextSearchMatchQuery` (each `EncryptFrom` and + `Into`), so it carries only the term its operator reads: + `=` and `<` on two-letter text work, and a match query on it is refused. + The by-name dispatch picks the projection for the index it is handed. +- **The block ORE types are producible** (`stack-encrypt` feature): + `IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre` and + `DoubleOrdOre` from `i32`, `i16`, `i64`, `f32` and `f64`, and `TextOrdOre` and `TextSearchOre` from + `String`. Their `ob` term is Stack Encrypt's block ORE term (ore-rs's + 6-bit fixed-width scheme for the numbers, its chained scheme for text): + one element, the hex of the whole ciphertext. `OreBlock256` implements + `EncryptFrom` for any orderable plaintext and never decrypts. This is not + EQL v3's `ob` (cipherstash-client's 8-bit blocks, up to six per text + value), and only EQL v4's comparator, `compare_ore_block_v2_term`, reads + it. Every term carries a random nonce, so a query's `ob` never equals the + stored one; the comparator orders them. A query carries the full + ciphertext. `TextSearchOre` follows `TextSearch`: its query carries only + the term its operator reads. Text whose fold is longer than 1023 bytes or + ends in NUL has no block ORE term and is refused. In the target table an + `ob` term is the index `ore_block`, not `ore`, which is CLLW ORE. +- **`encryption::targets::query` takes the index that answers the query's + operator** (`targets` feature), an `&IndexSpec` after the name, as Stack + Encrypt's `TargetResolver::query` hands it over. An index the type does + not carry is refused with `TargetError::NoIndex`, before anything is + derived. - **`Target::match_options`** (`targets` feature): the options a type's match index is built under, `EqlMatch`'s, for a type that carries one (`TextMatch`), and `None` otherwise. Stack Encrypt answers `matches` @@ -110,6 +177,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 separately from `all()` (which stays the stored + SteVec conversion-target inventory). +### Breaking + +- **`v4::text::TextSearchQuery` and `TextSearchOreQuery` hold each term as + an `Option`**, left out of the JSON when `None`, to match EQL v4's + `query_text_search` and `query_text_search_ore`, which now take any one + of their terms (ADR-0002, amended 2026-10-19). `pub hm: Hmac256` is now + `pub hm: Option`, and likewise `op`/`ob` and `bf`: code that + builds one of these structs wraps each term in `Some`, code that reads + one handles `None`, and a JSON operand missing a term now deserializes + where it was refused. `TextSearch` and `TextSearchOre`'s `bf` is + `SearchBloomFilter`, not `BloomFilter` (`.0` reaches the `BloomFilter`). + No published release of this crate has the `v4` module (3.0.6 does not), + so this breaks only code built against unreleased sources; the v3 twins + are unchanged, as cipherstash-client's query carries every term. + ### Changed - **`TextEq` and `TextEqQuery` compare text in Unicode 16 NFC** diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 4fdca915a..1146823b4 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -112,13 +112,13 @@ plaintext, or, for a date, a timestamp or a decimal, one of | v4 type | Plaintext | | --- | --- | -| `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe` | `String` | -| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe` | `i16` | -| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe` | `i32` | -| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe` | `i64` | +| `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe`, `TextOrdOre`, `TextSearch`, `TextSearchOre` | `String` | +| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | `i16` | +| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | `i32` | +| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | `i64` | | `Boolean` | `bool` | -| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe` | `f32` | -| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe` | `f64` | +| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `f32` | +| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `f64` | | `Date`, `DateEq` | `v4::plaintext::Date`: an `i32` of days since 1970-01-01 | | `Timestamp`, `TimestampEq` | `v4::plaintext::Timestamp`: `i64` Unix seconds and `u32` nanoseconds, UTC, no leap seconds | | `Numeric`, `NumericEq` | `v4::plaintext::Decimal`: an `i128` mantissa (below 2^96) and a `u8` scale (at most 28) | @@ -162,11 +162,31 @@ where that matters. Write the stored column on the left, `col @@ query`: the other order finds nothing until #1171 is fixed. Text too short to make a 3-gram (empty strings, two-letter codes) cannot be stored in a `TextMatch` column yet, and a `TextMatchQuery` for it is refused too: one filter type -builds both, and an empty filter as a query would match every row. The ORE -types (`*OrdOre`, `TextSearchOre`) wait for block ORE (#1132), `TextSearch` -for how a query that is not a match leaves out its match term (#1131), -`Date`, `Timestamp` and `Numeric` for a choice of their Rust plaintext types -(#1130), and `Json` for a decision on sealing a document as one leaf. +builds both, and an empty filter as a query would match every row. +`TextSearch` stores all three terms (`TextEq`'s `hm`, `TextOrd`'s `op` and +`TextMatch`'s `bf`). Text too short for a 3-gram is stored with the empty +filter (`"bf": []`, the `SearchBloomFilter` field type): `=` and `<` find it +through `hm` and `op`, and no `@@` query does, because a query's filter is +never empty. (A `@@` between two stored values, which no producer builds, is +not a query: two stored empty filters match each other there, as in v3.) Its query carries only the term its operator reads: build it +through `TextSearchEqualityQuery` (`=`, `<>`), `TextSearchOrderQuery` (`<`, +`<=`, `>`, `>=`) or `TextSearchMatchQuery` (`@@`) and convert into +`TextSearchQuery`, and the by-name dispatch picks the one for the index that +answers the operator. So `=` and `<` work on short text, a match query on it +is refused, and EQL v4 raises on an operator whose term the query lacks (or +holds as `null`, or as an empty filter). +The ORE types (`*OrdOre`, `TextSearchOre`) store a block ORE term (`ob`): +one element, the hex of a whole ore-rs 6-bit block ORE ciphertext (the +fixed-width scheme for numbers, the chained scheme for text, padded to a +power-of-two bucket of at least 16 bytes). They order by the same key as the +`Ord` types, but EQL v4 compares them with its own comparator, +`eql_v4_internal.compare_ore_block_v2_term`, and `ORDER BY` reads +`eql_v4.ord_term_ore(col)`. Every term carries a random nonce, so a query's +`ob` never equals the stored bytes; the comparator orders them. Text whose +fold is longer than 1023 bytes or ends in NUL has no block ORE term and is +refused. `TextSearchOre` follows `TextSearch`, with `ob` in place of `op`. +`Date`, `Timestamp` and `Numeric` wait for a choice of their Rust plaintext +types, and `Json` for a decision on sealing a document as one leaf. Each ordered v4 family also has two range query types, one per order term: `IntegerRangeOreQuery` and `IntegerRangeOpeQuery`, and so on @@ -229,6 +249,71 @@ descriptor is `/`, two parts, the same context and descriptor as a two-segment `stack_encrypt::Label`. A table or column containing `/` is escaped in the descriptor rather than read as two names. +### Querying an EQL v4 field from Rust + +A query names an operator, and a plan for the column's EQL type finds the +index that answers it (Stack Encrypt's ADR-0009). The term that comes back +is the type's query twin, the value the `eql_v4.query_*` domain takes: + +```rust,ignore +use eql_bindings::v4::text::{TextEq, TextMatch}; +use stack_encrypt::op::{self, Probabilistic, Stats}; +use stack_encrypt::Plan; + +// A plan for the column's type, under the label that names the column: +// "users/email" is Identifier::for_column("users", "email"). +let email_plan = Plan::value::().context("users/email").encrypt_into::().build()?; +let query = cipher.query(op::Eq, "alice@example.com").on(&email_plan).await?; // a TextEqQuery +// WHERE email = $1::eql_v4.query_text_eq, binding sqlx::types::Json(&query) + +let name_plan = Plan::value::().context("users/name").encrypt_into::().build()?; +let query = cipher.query(op::Matches, "ali").on(&name_plan).await?; // a MatchTwin +let fetch = query.target_limit(10, &Stats::declared(1_000_000)).fetch_or(50); +// WHERE name @@ $1::eql_v4.query_text_match LIMIT $2, then keep the +// decrypted rows for which query.verify(&name) is true. +``` + +| v4 type | Operator | Query term | +| --- | --- | --- | +| `TextEq` | `op::Eq` | `TextEqQuery` | +| `SmallintEq`, `IntegerEq`, `BigintEq`, `RealEq`, `DoubleEq` | `op::Eq` | `SmallintEqQuery`, `IntegerEqQuery`, `BigintEqQuery`, `RealEqQuery`, `DoubleEqQuery` | +| `DateEq`, `TimestampEq`, `NumericEq` | `op::Eq` | `DateEqQuery`, `TimestampEqQuery`, `NumericEqQuery` (from `v4::plaintext::Date`, `Timestamp`, `Decimal`) | +| `TextMatch` | `op::Matches` | `MatchTwin` | + +`MatchTwin` serializes as the twin and adds `verify` and `target_limit` +under `EqlMatch`: a filter can match a row that does not hold the query's +text, so keep only the decrypted rows that verify. An exact query has no +`verify`. An operator a type does not answer does not compile: `op::Lt` on +`TextEq`, or `op::Matches` on `IntegerEq`. Nor does naming another index +with `.via(..)`, such as the bare `Equality`, whose term the type's SQL does +not take. The ordered types (`IntegerOrd`, `TextOrd`, …) answer nothing in +the typed chain yet: their twins need an order term, or several terms at +once, and the chain sends one index's term. Query them through the data +path (`stack_encrypt::dynamic::query_term`) for now. + +The label must name the column the value was written under, as two +segments, table then column. A label of any other length, and a query +extended with `.extend(..)` (whose terms would be derived under more than +the column the twin stores), are refused when the query is awaited, before +any keyset is resolved, with an `encryption::query::QueryLabelError` in +`Error::Other`. A label's segments are plain, so a table or column name +that is not a plain label segment cannot be named on a plan, and such a +column, which `Identifier::for_column` accepts, cannot be queried this way: +for instance a name that starts with a digit, `-` or `b64:`, or one holding +`/`, `(`, `)`, or a control or invisible format character. +The plan is for queries: write with +`keyset.encrypt_as(&value, column)` and open with `cipher.decrypt_as`, since +an EQL type is sealed under its column rather than the plan's label. The +query is the same bytes the data path derives for a field that names the +type (`stack_encrypt::dynamic::query_term`), which is what the Go SDK sends, +and `tests/encryption/tests/typed_queries.rs` checks that for every type and +operator. The `op::Answers` impls are generated into `src/v4/answers.rs` +from the catalog by the rule the data path uses, and need only the +`stack-encrypt` feature, not `targets`; the supporting types are in +`eql_bindings::encryption::query`, whose +[executable example](src/encryption/query_example.rs) runs with +`cargo test -p eql-encryption-tests --test query_example`. + ### Interoperability matrix The plan asked for the three producer/reader pairings to be proven rather than diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index ee0c14727..457ae3fb5 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -89,6 +89,8 @@ //! require::(); //! ``` +pub mod query; + // The by-name dispatch over the EQL v4 types (`targets` feature). A plain // comment: a doc comment here would resolve the module's own links from // this scope. @@ -99,10 +101,10 @@ use base64::{engine::general_purpose::STANDARD, Engine as _}; #[allow(deprecated)] use stack_encrypt::sem::MatchTerm; use stack_encrypt::sem::{ - EqualityTerm, MatchConfig, MatchOptions, OpeTerm, Orderable, TermError, Tokenizer, + EqualityTerm, MatchConfig, MatchOptions, OpeTerm, Orderable, OreBlockTerm, TermError, Tokenizer, }; use stack_encrypt::target::transcode::{Reader, Transcode, Visitor}; -use stack_encrypt::target::{self, AeadContext, CallerContext}; +use stack_encrypt::target::{self, AeadContext, CallerContext, IndexSpec}; use stack_encrypt::{ CipherText, ContextPiece, Decrypt, DecryptField, DecryptInto, Decryptable, Decryption, Describe, Description, Encrypt, EncryptFrom, Encryption, Error, IntoContext, MaybeEmpty, @@ -110,7 +112,8 @@ use stack_encrypt::{ }; use crate::{ - v3::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw}, + v3::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256}, + v4::terms::SearchBloomFilter, Identifier, }; @@ -134,6 +137,20 @@ impl Identifier { c: column.into(), }) } + + /// The column a two-segment label names, table then column, or `None` + /// for a label of any other length. One reading of a label for the + /// by-name dispatch (`targets::Identifier::from_label`) and the typed + /// query chain ([`query`]). + pub(crate) fn column_of(label: &stack_encrypt::Label) -> Option> { + let mut segments = label.segments(); + let (Some(table), Some(column), None) = (segments.next(), segments.next(), segments.next()) + else { + return None; + }; + // A label segment is plain, so never empty. + Self::for_column(table, column).ok() + } } impl MaybeEmpty for Identifier { @@ -242,9 +259,15 @@ pub struct EqualityVisitor; impl Visitor for EqualityVisitor { type Value = Hmac256; fn equality(self, term: EqualityTerm) -> Result { - Ok(Hmac256(hex::encode(term.as_bytes()))) + Ok(hmac256(&term)) } } + +/// EQL's `hm` for an equality term: the HMAC, hex encoded. One encoding for +/// stored values and query twins alike. +fn hmac256(term: &EqualityTerm) -> Hmac256 { + Hmac256(hex::encode(term.as_bytes())) +} impl Transcode for Hmac256 { type Visitor = EqualityVisitor; fn visitor() -> Self::Visitor { @@ -259,6 +282,11 @@ impl EncryptFrom for Hmac256 { { target::equality().transcode() } + /// The equality index: a plan for a type holding an `hm` answers + /// `op::Eq` through it. + fn indexes() -> Vec { + vec![IndexSpec::Equality] + } } impl Decryptable for Hmac256 { const DECRYPTABLE: bool = false; @@ -304,11 +332,12 @@ impl MatchConfig for EqlMatch { /// /// Text that cuts no token (empty, separator-only, or shorter than a 3-gram) /// is refused with [`TermError::EmptyTermText`], stored or queried. Stack -/// Encrypt stores such a value as the empty filter, but EQL treats an empty -/// filter as contained by every stored filter, and this one type builds both -/// the stored filter and the query operand, so it cannot tell a query from a -/// stored value: refusing both is the one way a query cannot match every -/// row. +/// Encrypt stores such a value as the empty filter, but this type is the +/// filter of every match query, and a query's filter must never be empty; +/// it is also `TextMatch`'s stored filter, and a `TextMatch` value with an +/// empty filter is a row no operator can find. A search domain's stored +/// filter, which `=` and `<` find through its other terms, is +/// [`SearchBloomFilter`] and takes the empty filter ([`SearchMatchVisitor`]). #[doc(hidden)] pub struct MatchVisitor; impl Visitor for MatchVisitor { @@ -321,22 +350,27 @@ impl Visitor for MatchVisitor { if terms.positions().is_empty() { return Err(Error::Term(TermError::EmptyTermText)); } - terms - .positions() - .iter() - .map(|&position| { - i16::try_from(position).map_err(|_| { - Error::Other( - "a match position does not fit a smallint; EQL's bloom filter stores \ - filters of at most 32768 bits" - .into(), - ) - }) - }) - .collect::>() - .map(BloomFilter) + bloom_filter(terms.positions()) } } + +/// EQL's `bf` for a filter's set positions, each as a `smallint`. One +/// encoding for stored values and query twins alike. +fn bloom_filter(positions: &[u16]) -> Result { + positions + .iter() + .map(|&position| { + i16::try_from(position).map_err(|_| { + Error::Other( + "a match position does not fit a smallint; EQL's bloom filter stores \ + filters of at most 32768 bits" + .into(), + ) + }) + }) + .collect::>() + .map(BloomFilter) +} impl Transcode for BloomFilter { type Visitor = MatchVisitor; fn visitor() -> Self::Visitor { @@ -357,6 +391,11 @@ impl> EncryptFrom for BloomFilter { let terms: Encryption<'s, S, MatchTerm, K, CallerContext> = target::matching(); terms.transcode() } + /// The match index under [`EqlMatch`]: a plan for a type holding a + /// `bf` answers `op::Matches` through it. + fn indexes() -> Vec { + vec![IndexSpec::Match(EqlMatch::options())] + } } impl Decryptable for BloomFilter { const DECRYPTABLE: bool = false; @@ -367,6 +406,50 @@ impl DecryptField for BloomFilter { } } +/// Builds a stored search value's filter: [`MatchVisitor`]'s filter, except +/// that text which cuts no token stores the empty filter instead of being +/// refused (see [`SearchBloomFilter`]). +#[doc(hidden)] +pub struct SearchMatchVisitor; +impl Visitor for SearchMatchVisitor { + type Value = SearchBloomFilter; + // As in `MatchVisitor`: the one spelling of the terms' type that compiles + // against both the crates.io stack-encrypt and the tree. + #[allow(deprecated)] + fn matching(self, terms: MatchTerm) -> Result { + if terms.positions().is_empty() { + return Ok(SearchBloomFilter(BloomFilter(Vec::new()))); + } + MatchVisitor.matching(terms).map(SearchBloomFilter) + } +} +impl Transcode for SearchBloomFilter { + type Visitor = SearchMatchVisitor; + fn visitor() -> Self::Visitor { + SearchMatchVisitor + } +} +impl> EncryptFrom for SearchBloomFilter { + type Context = CallerContext; + fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> + where + S: 's, + { + // Configuration named by the type, as for `BloomFilter`. + #[allow(deprecated)] + let terms: Encryption<'s, S, MatchTerm, K, CallerContext> = target::matching(); + terms.transcode() + } +} +impl Decryptable for SearchBloomFilter { + const DECRYPTABLE: bool = false; +} +impl DecryptField for SearchBloomFilter { + fn decryption_field(self, _: Ctx) -> Option> { + None + } +} + /// Builds EQL's hex OPE term from the native order-preserving term. /// /// The bytes are Stack Encrypt's CLLW OPE ciphertext of the value's @@ -412,6 +495,52 @@ impl DecryptField for OpeCllw { } } +/// Builds EQL v4's `ob` from the native block ORE term: one element, the +/// hex of the whole ore-rs ciphertext. +/// +/// The ciphertext is ore-rs's 6-bit block ORE of the value's ADR-0002 order +/// key (`OreBlockTerm`): the fixed-width scheme for integers and floats, the +/// chained one, padded to a power-of-two length bucket of at least 16 +/// bytes, for text. EQL v4 compares `ob` with the comparator for that +/// format, which takes exactly one element; EQL v3's `ob` holds 8-bit block +/// terms, up to six for text, that cipherstash-client writes. A query +/// carries the full ciphertext too: the comparator reads the right half of +/// the operand a btree index or `between` passes second. +/// +/// Text with no fold, text whose fold ends in NUL or is longer than 1023 +/// bytes has no block ORE term and is refused, stored or queried. +#[doc(hidden)] +pub struct OreBlockVisitor; +impl Visitor for OreBlockVisitor { + type Value = OreBlock256; + fn ore_block(self, term: OreBlockTerm) -> Result { + Ok(OreBlock256(vec![hex::encode(term.as_bytes())])) + } +} +impl Transcode for OreBlock256 { + type Visitor = OreBlockVisitor; + fn visitor() -> Self::Visitor { + OreBlockVisitor + } +} +impl EncryptFrom for OreBlock256 { + type Context = CallerContext; + fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> + where + S: 's, + { + target::ore_block().transcode() + } +} +impl Decryptable for OreBlock256 { + const DECRYPTABLE: bool = false; +} +impl DecryptField for OreBlock256 { + fn decryption_field(self, _: Ctx) -> Option> { + None + } +} + #[cfg(test)] mod tests { use stack_encrypt::{nonempty, Descriptor, Label}; diff --git a/packages/eql/crates/eql-bindings/src/encryption/query.rs b/packages/eql/crates/eql-bindings/src/encryption/query.rs new file mode 100644 index 000000000..da22a1736 --- /dev/null +++ b/packages/eql/crates/eql-bindings/src/encryption/query.rs @@ -0,0 +1,433 @@ +//! Querying an EQL v4 field through Stack Encrypt's typed query chain. +//! +//! A query names an operator, and the plan it runs on finds the index that +//! answers it (Stack Encrypt's ADR-0009): +//! `cipher.query(op::Eq, "bob@example.com").on(&plan)`. On a plan built for +//! an EQL v4 type, `Plan::value::().encrypt_into::()`, the +//! type says which operators it answers through its +//! [`op::Answers`] impls, and the term that comes back is the type's +//! *query twin*: the value of the `eql_v4.query_*` domain the SQL compares +//! against the column. +//! +//! | type | operator | query term | +//! |---|---|---| +//! | `TextEq` | [`op::Eq`](struct@op::Eq) | `TextEqQuery` | +//! | `IntegerEq` | [`op::Eq`](struct@op::Eq) | `IntegerEqQuery` | +//! | `BigintEq` | [`op::Eq`](struct@op::Eq) | `BigintEqQuery` | +//! | `RealEq` | [`op::Eq`](struct@op::Eq) | `RealEqQuery` | +//! | `DoubleEq` | [`op::Eq`](struct@op::Eq) | `DoubleEqQuery` | +//! | `TextMatch` | [`op::Matches`](struct@op::Matches) | [`MatchTwin`](MatchTwin) | +//! +//! eql-codegen writes those impls from the catalog, by the rule Stack +//! Encrypt's data path answers an EQL type's operators with: `Eq` through +//! the equality index, `Matches` through the match index. Each type is also +//! an [`op::LabelledTarget`], which is what lets its plan be queried. The +//! ordered types (`IntegerOrdOre`, `TextOrd` and the rest) are not produced +//! yet, so they answer nothing. An operator a type does not answer does not +//! compile, and neither does naming another index with `.via(..)`: the +//! plan sends the type's twin or nothing. +//! +//! The query is derived under the plan's label, which must name the column +//! the value was written under: a two-segment label, table then column, is +//! the [`Identifier`] the value stores, so +//! `"users/email"` queries values written with +//! `Identifier::for_column("users", "email")`. A label's segments are plain +//! ([`Label`]): none may start with a digit or `-`, for +//! instance, though `Identifier::for_column` takes such a name. A column +//! whose table or column name is not a plain segment can be written but +//! not named on a plan, so it cannot be queried this way. A label of any other length +//! names no column, and a query extended with `.extend(..)` is derived under +//! more than the column the twin stores, so the awaited query refuses both, +//! before any keyset is resolved or term derived. +//! +//! The plan is for queries. Stack Encrypt runs a plan's encryption under +//! the label, and an EQL type is sealed under its column instead, so write +//! with `keyset.encrypt_as(&value, column)` and open with +//! `cipher.decrypt_as(stored, column.into())`, as the +//! [encryption example](super#example) does. +//! +//! The query is the bytes Stack Encrypt's data path derives for a field +//! that names the same type (`stack_encrypt::dynamic::query_term`), which +//! is what the Go SDK sends. +//! +//! # Example +//! +//! The code below is executed by `cargo test -p eql-encryption-tests --test +//! query_example`, for the reason the [encryption example](super#example) +//! gives. +#![doc = concat!("```rust,ignore\n", include_str!("query_example.rs"), "\n```")] +//! +//! # Operators a type does not answer +//! +//! A plan for `TextEq` answers `Eq`: +//! ```no_run +//! use eql_bindings::v4::text::TextEq; +//! use stack_encrypt::plan::{Typed, ValuePlan}; +//! use stack_encrypt::{op, StackCipher}; +//! fn query(cipher: &StackCipher, plan: &ValuePlan>) { +//! let _ = cipher.query(op::Eq, "bob").on(plan); +//! } +//! ``` +//! and no ordering, since `TextEq` carries no order term: +//! ```compile_fail +//! use eql_bindings::v4::text::TextEq; +//! use stack_encrypt::plan::{Typed, ValuePlan}; +//! use stack_encrypt::{op, StackCipher}; +//! fn query(cipher: &StackCipher, plan: &ValuePlan>) { +//! let _ = cipher.query(op::Lt, "bob").on(plan); +//! } +//! ``` +//! A plan for `IntegerEq` answers `Eq`: +//! ```no_run +//! use eql_bindings::v4::integer::IntegerEq; +//! use stack_encrypt::plan::{Typed, ValuePlan}; +//! use stack_encrypt::{op, StackCipher}; +//! fn query(cipher: &StackCipher, plan: &ValuePlan>) { +//! let _ = cipher.query(op::Eq, &42).on(plan); +//! } +//! ``` +//! and no text match: +//! ```compile_fail +//! use eql_bindings::v4::integer::IntegerEq; +//! use stack_encrypt::plan::{Typed, ValuePlan}; +//! use stack_encrypt::{op, StackCipher}; +//! fn query(cipher: &StackCipher, plan: &ValuePlan>) { +//! let _ = cipher.query(op::Matches, &42).on(plan); +//! } +//! ``` +//! Nor does a plan for `TextEq` take another index, whose term the type's +//! SQL would not take, such as the bare equality index: +//! ```compile_fail +//! use eql_bindings::v4::text::TextEq; +//! use stack_encrypt::plan::{Typed, ValuePlan}; +//! use stack_encrypt::{op, Equality, StackCipher}; +//! fn query(cipher: &StackCipher, plan: &ValuePlan>) { +//! let _ = cipher.query(op::Eq, "bob").via(Equality).on(plan); +//! } +//! ``` +//! or another type's twin index, which answers that type's plaintext only: +//! ```compile_fail +//! use eql_bindings::encryption::query::TwinIndex; +//! use eql_bindings::v4::integer::IntegerEqQuery; +//! use eql_bindings::v4::text::TextEq; +//! use stack_encrypt::plan::{Typed, ValuePlan}; +//! use stack_encrypt::{op, Equality, StackCipher}; +//! fn query(cipher: &StackCipher, plan: &ValuePlan>) { +//! let index = TwinIndex::::default(); +//! let _ = cipher.query(op::Eq, "bob").via(index).on(plan); +//! } +//! ``` +//! The twin index answers for its twin's own plaintext only +//! ([`QueryTwin::Plaintext`]): `TextEqQuery`'s for a `String`, +//! ```no_run +//! use eql_bindings::encryption::query::TwinIndex; +//! use eql_bindings::v4::text::TextEqQuery; +//! use stack_encrypt::{op, Equality}; +//! fn answers>() {} +//! answers::>(); +//! ``` +//! and not `IntegerEqQuery`'s, whose plaintext is an `i32`: +//! ```compile_fail +//! use eql_bindings::encryption::query::TwinIndex; +//! use eql_bindings::v4::integer::IntegerEqQuery; +//! use stack_encrypt::{op, Equality}; +//! fn answers>() {} +//! answers::>(); +//! ``` +//! A `TextMatch` query verifies a row; a `TextEq` query is exact, so it has +//! nothing to verify and no `verify`: +//! ```compile_fail +//! use eql_bindings::v4::text::TextEqQuery; +//! use stack_encrypt::op::Probabilistic; +//! fn rows(query: &TextEqQuery) -> bool { +//! query.verify("bob") +//! } +//! ``` +//! +//! [`op::Answers`]: stack_encrypt::op::Answers +//! [`op::LabelledTarget`]: stack_encrypt::op::LabelledTarget + +use std::fmt; +use std::marker::PhantomData; + +use serde::{Serialize, Serializer}; +use stack_encrypt::kms::MaybeSend; +use stack_encrypt::op::{ + self, CaseInsensitive, Limit, MatchCase, Probabilistic, QueryTerm, QueryToken, Stats, Supports, +}; +use stack_encrypt::sem::{EqualityTerm, MatchQuery, MatchTerms, QueryTokens}; +use stack_encrypt::target::{CallerContext, ConsumeSource, Encryption, Index, IndexSpec}; +use stack_encrypt::{Encrypt, Equality, Error, Label, Match, NonEmpty}; + +use super::{bloom_filter, hmac256, EqlMatch}; +use crate::v3::terms::{BloomFilter, Hmac256}; +use crate::Identifier; + +/// EQL's match configuration folds case, so its index answers the +/// case-insensitive [`op::Matches`](struct@op::Matches). +impl MatchCase for EqlMatch { + type Case = CaseInsensitive; +} + +/// An EQL v4 query twin, assembled from the column it queries and the one +/// term its type's index derived for the query's value. eql-codegen +/// implements it for the twin of every type that answers an operator +/// ([`TextEqQuery`] from an [`Hmac256`], [`TextMatchQuery`] from a +/// [`BloomFilter`]), with the version every v4 value carries. +/// +/// [`TextEqQuery`]: crate::v4::text::TextEqQuery +/// [`TextMatchQuery`]: crate::v4::text::TextMatchQuery +pub trait QueryTwin: Sized { + /// The plaintext the twin's type is produced from: `String` for + /// `TextEqQuery`, `i32` for `IntegerEqQuery`. A [`TwinIndex`] answers + /// for this plaintext only, so no plan can send one type's twin for + /// another type's value. + type Plaintext; + /// The twin of `term`, derived under `column`. + fn assemble(column: Identifier, term: T) -> Self; +} + +/// The index an EQL v4 type answers an operator through: Stack Encrypt's +/// index `I`, whose query term is sent as the type's query twin `Q`. +/// +/// It derives exactly what `I` derives, under the plan's label, and checks +/// and refuses what `I` refuses, as well as a label that names no column +/// and any extension; it then writes the term into the twin, with the +/// column the label names. It answers for the twin's own plaintext +/// ([`QueryTwin::Plaintext`]) only. `TextEq` answers [`op::Eq`](struct@op::Eq) through +/// `TwinIndex`, and `TextMatch` answers +/// [`op::Matches`](struct@op::Matches) through +/// `TwinIndex, TextMatchQuery>`. +pub struct TwinIndex { + index: I, + twin: PhantomData Q>, +} + +impl Default for TwinIndex { + fn default() -> Self { + Self { + index: I::default(), + twin: PhantomData, + } + } +} + +impl Clone for TwinIndex { + fn clone(&self) -> Self { + Self { + index: self.index.clone(), + twin: PhantomData, + } + } +} + +impl fmt::Debug for TwinIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TwinIndex") + .field("index", &self.index) + .field("twin", &std::any::type_name::()) + .finish() + } +} + +/// The index `I`, unchanged: the same spec, so a plan holds it as it holds +/// `I`, and the same derivation, so its term is `I`'s. +impl, Q> Index for TwinIndex { + type Term = I::Term; + fn spec(&self) -> IndexSpec { + self.index.spec() + } + fn operation<'s, K: 'static, M: ConsumeSource<'s, S>>( + &self, + ) -> Encryption<'s, S, Self::Term, K, CallerContext, M> + where + S: 's, + { + self.index.operation() + } +} + +/// Refuse a query context no EQL query value can record: a label that +/// names no column (it must be two segments, table then column), or any +/// extension (`.extend(..)`), since the terms would be derived under the +/// column and the extension while the twin stores the column alone, and +/// would match nothing. Before any keyset is resolved or term derived. +fn check_column(label: &Label, extended: bool) -> Result<(), Error> { + if Identifier::column_of(label).is_none() { + return Err(Error::Other(Box::new(QueryLabelError::NotAColumn { + label: label.to_string(), + }))); + } + if extended { + return Err(Error::Other(Box::new(QueryLabelError::Extended))); + } + Ok(()) +} + +/// Why a typed query on an EQL v4 type was refused for its context, before +/// any keyset was resolved. It arrives as [`Error::Other`]; match it with +/// `downcast_ref::()`. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum QueryLabelError { + /// The label names no column: an EQL query value stores the + /// [`Identifier`] it queries, a two-segment label, table then column. + /// The data path refuses the same label as `TargetError::Context`. + NotAColumn { + /// The label, rendered. + label: String, + }, + /// The query was extended with `.extend(..)`, so its terms would be + /// derived under more than the column the twin stores, and would match + /// nothing. + Extended, +} + +impl fmt::Display for QueryLabelError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotAColumn { label } => write!( + f, + "{label:?} is not an EQL column identifier: expected two segments, table and column" + ), + Self::Extended => f.write_str( + "an EQL query value stores the column it queries and nothing else, so its query \ + cannot be extended", + ), + } + } +} + +impl std::error::Error for QueryLabelError {} + +/// The column a query's label names. The chain calls `query` only after +/// `check_label` accepted the label (`Supports::query`), and every +/// `Supports` impl here delegates `check_label` to [`check_column`], so the +/// label is two segments; a label that is not would be a broken invariant, +/// and fails here rather than as a twin with an empty column. +fn column(token: &QueryToken) -> Identifier { + Identifier::column_of(token.label()) + .map(NonEmpty::into_inner) + .expect("check_label accepted the query's label as a column before any term was derived") +} + +/// Equality: the twin's `hm` is the HMAC the equality index derives, hex +/// encoded as the stored value's is, so the twin equals the stored value's +/// term under the same column. Only for the plaintext the twin's type is +/// produced from. +impl Supports for TwinIndex +where + Q: QueryTwin + QueryTerm + MaybeSend + 'static, + Q::Plaintext: Encrypt + 'static, +{ + type Query = Q; + type Verifier = (); + fn check(&self, value: &Q::Plaintext) -> Result<(), Error> { + Supports::::check(&self.index, value) + } + fn check_label(&self, label: &Label, extended: bool) -> Result<(), Error> { + check_column(label, extended) + } + fn verifier(&self, _: &Q::Plaintext) {} + fn query((): (), term: EqualityTerm, token: QueryToken) -> Q { + Q::assemble(column(&token), hmac256(&term)) + } +} + +/// Text match: the twin's `bf` is the filter EQL's match index derives +/// over the query's text, and the query keeps that text's tokens to +/// [`verify`](Probabilistic::verify) each decrypted row, as Stack Encrypt's +/// `MatchQuery` does. A query whose text cuts no token is refused before +/// any term exists. Only for the plaintext the twin's type is produced +/// from. +impl Supports for TwinIndex, Q> +where + Q: QueryTwin + MaybeSend + 'static, + Q::Plaintext: AsRef + 'static, +{ + type Query = MatchTwin; + type Verifier = QueryTokens; + fn check(&self, value: &Q::Plaintext) -> Result<(), Error> { + Supports::::check(&self.index, value) + } + fn check_label(&self, label: &Label, extended: bool) -> Result<(), Error> { + check_column(label, extended) + } + fn verifier(&self, value: &Q::Plaintext) -> QueryTokens { + Supports::::verifier(&self.index, value) + } + fn admit(&self, value: &Q::Plaintext) -> Result { + Supports::::admit(&self.index, value) + } + fn query(tokens: QueryTokens, terms: MatchTerms, token: QueryToken) -> MatchTwin { + let column = column(&token); + let query = + as Supports>::query(tokens, terms, token); + // `EqlMatch`'s filter is 2048 bits, so every position fits a + // smallint (`the_match_configuration_is_pinned`); and `admit` + // refused a query with no tokens, so the filter is not empty. + let bf = bloom_filter(query.positions()) + .expect("EqlMatch's 2048-bit filter keeps every position within a smallint"); + MatchTwin { + twin: Q::assemble(column, bf), + query, + } + } +} + +/// A text match query on an EQL v4 type: the type's query twin `Q`, which +/// is what the SQL binds, and the query's tokens, which decide whether a +/// row the server returned is a real match. +/// +/// A Bloom filter can hold every bit of the query without the value holding +/// the query's text, so the server's answer can include rows that do not +/// match, and never misses one that does. [`verify`](Probabilistic::verify) +/// checks a decrypted value against the query's tokens, under EQL's match +/// configuration ([`EqlMatch`]), and +/// [`target_limit`](Probabilistic::target_limit) says how many rows to fetch +/// to expect a number of real matches. Both answer exactly as Stack +/// Encrypt's data path answers for the same query (`dynamic::verify` and +/// `dynamic::target_limit`). +/// +/// It serializes as the twin, so it binds as one: `sqlx::types::Json(&query)` +/// with `$1::eql_v4.query_text_match`. The tokens are plaintext: they are +/// never serialized, never appear in `Debug`, and are zeroized when the +/// query is dropped. +#[derive(Clone, Debug, PartialEq)] +pub struct MatchTwin { + twin: Q, + query: MatchQuery, +} + +impl MatchTwin { + /// The query twin, to bind. + pub fn twin(&self) -> &Q { + &self.twin + } + + /// The query twin, without the tokens that verify rows. + pub fn into_twin(self) -> Q { + self.twin + } +} + +impl QueryTerm for MatchTwin {} + +impl Probabilistic for MatchTwin { + type Plaintext = str; + + fn verify(&self, value: &str) -> bool { + self.query.verify(value) + } + + fn target_limit(&self, want: u64, stats: &Stats) -> Limit { + self.query.target_limit(want, stats) + } +} + +/// The twin alone: what the server compares. +impl Serialize for MatchTwin { + fn serialize(&self, serializer: Ser) -> Result { + self.twin.serialize(serializer) + } +} diff --git a/packages/eql/crates/eql-bindings/src/encryption/query_example.rs b/packages/eql/crates/eql-bindings/src/encryption/query_example.rs new file mode 100644 index 000000000..284c6cef4 --- /dev/null +++ b/packages/eql/crates/eql-bindings/src/encryption/query_example.rs @@ -0,0 +1,51 @@ +use eql_bindings::{ + v4::text::{TextEq, TextMatch}, + Identifier, +}; +use stack_encrypt::op::{self, Probabilistic, Stats}; +use stack_encrypt::{Plan, StackCipher}; +use stack_kms::FakeDataKeySource; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + // Local example only: these keys are held in memory, not in ZeroKMS. + let cipher = StackCipher::builder() + .kms(FakeDataKeySource::new()) + .init() + .await?; + let keyset = cipher.default_keyset(); + + // Write a value to the `users.email` column. + let column = Identifier::for_column("users", "email")?; + let email = String::from("alice@example.com"); + let stored: TextEq = keyset.encrypt_as(&email, column).await?; + + // Query it: a plan for the column's type, under the label that names + // the same column, table then column. + let email_plan = Plan::value::() + .context("users/email") + .encrypt_into::() + .build()?; + // A `TextEqQuery`: bind it as `$1::eql_v4.query_text_eq`. + let query = cipher + .query(op::Eq, "alice@example.com") + .on(&email_plan) + .await?; + assert_eq!(query.hm, stored.hm); + assert_eq!(query.i, stored.i); + + // A text match: bind `query.twin()` (or the query itself, which + // serializes as the twin) as `$1::eql_v4.query_text_match`. + let name_plan = Plan::value::() + .context("users/name") + .encrypt_into::() + .build()?; + let query = cipher.query(op::Matches, "ali").on(&name_plan).await?; + // How many rows to fetch to expect 10 real matches in a million. + let _limit = query.target_limit(10, &Stats::declared(1_000_000)); + // The server can return rows that hold every bit of the filter without + // the text: keep the decrypted rows that verify. + assert!(query.verify("Alice Nguyen")); + assert!(!query.verify("Bob Li")); + Ok(()) +} diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 5eb2f91a8..79912347a 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -90,7 +90,7 @@ use std::fmt; use serde::{de::DeserializeOwned, Serialize}; use stack_encrypt::dynamic::{ValueFrame, ValueKind}; use stack_encrypt::kms::MaybeSend; -use stack_encrypt::target::ExpectedContext; +use stack_encrypt::target::{ExpectedContext, IndexSpec}; use stack_encrypt::{ Decrypt, DecryptInto, Encrypt, EncryptFrom, Error, KeysetCipher, Label, NonEmpty, Pending, StackCipher, @@ -200,6 +200,15 @@ pub enum TargetError { /// The type's name. name: &'static str, }, + /// A query asked through an index the type does not carry + /// ([`Target::indexes`]), so it has no term to derive. + #[error("{name} carries no {index} index, so no query is answered through it")] + NoIndex { + /// The type's name. + name: &'static str, + /// The index asked for, by its `IndexSpec::key()`. + index: &'static str, + }, /// The field's context is not an EQL column: an [`Identifier`] is a /// two-segment label, table then column, and this label is not one. #[error("{label:?} is not an EQL column identifier: expected two segments, table and column")] @@ -280,17 +289,9 @@ impl Identifier { /// /// [`TargetError::Context`] for a label of any other length. pub fn from_label(label: &Label) -> Result, TargetError> { - let context = || TargetError::Context { + Identifier::column_of(label).ok_or_else(|| TargetError::Context { label: label.to_string(), - }; - let mut segments = label.segments(); - let (Some(table), Some(column), None) = (segments.next(), segments.next(), segments.next()) - else { - return Err(context()); - }; - // A label segment is plain, so never empty; the `Err` arm is the - // type's, not a case this function can reach. - Identifier::for_column(table, column).map_err(|_| context()) + }) } } @@ -344,21 +345,52 @@ pub fn decrypt<'a, K: 'static>( /// matches stored values of the type under `context`'s column, as JSON /// bytes. A query derives no data key, so the pending settles without I/O. /// +/// `index` is the type's index that answers the query's operator, one of +/// [`Target::indexes`]; Stack Encrypt passes the one its +/// `OpSpec::answered_by` picks. Most twins carry every term of their type +/// whatever the index. `TextSearchQuery` carries the index's term alone +/// (EQL v4's search query terms are optional), so `=` and `<` on text too +/// short for a match term are answered, and `matches` on it is refused. +/// /// # Errors /// -/// As [`encrypt`]. +/// As [`encrypt`], and [`TargetError::NoIndex`] for an index the type does +/// not carry. pub fn query<'a, K: 'static>( name: &str, + index: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, context: &Label, plaintext: ValueFrame, ) -> Result, K>, TargetError> { producible(name)?; - if let Some(error) = no_query(target(name)) { + let Some(target) = target(name) else { + return Err(refuse(name)); + }; + if let Some(error) = no_query(Some(target)) { return Err(error); } + carries(target, index)?; let column = Identifier::from_label(context)?; - query_named(name, keyset, column, plaintext) + query_named(name, index, keyset, column, plaintext) +} + +/// Refuse a query through an index the type does not carry. +fn carries(target: &'static Target, index: &IndexSpec) -> Result<(), TargetError> { + match target.indexes.iter().find(|key| **key == index.key()) { + Some(_) => Ok(()), + None => Err(no_index(target.name, index)), + } +} + +/// The refusal for a query on `name` through an index it does not carry. +/// Also the generated query dispatch's fall-through for a twin built per +/// index. +pub(crate) fn no_index(name: &'static str, index: &IndexSpec) -> TargetError { + TargetError::NoIndex { + name, + index: index.key(), + } } /// Refuse a name the dispatch has no arm for, as the table explains it. @@ -536,6 +568,32 @@ where .try_map(|value| serde_json::to_vec(&value).map_err(|error| Error::Other(Box::new(error))))) } +/// Run the plan of `P`, a projection of the query twin `Q` that carries one +/// term, and resolve to `Q`'s JSON bytes: the twin of a type whose query +/// terms are optional, holding only the term `P` derives. The generated +/// query dispatch picks `P` by the index that answers the query's operator. +pub(crate) fn run_projected<'a, P, Q, S, K>( + name: &'static str, + keyset: &'a KeysetCipher<'_, K>, + column: NonEmpty, + plaintext: ValueFrame, +) -> Result, K>, TargetError> +where + P: EncryptFrom> + Into, + Q: Serialize, + S: Plaintext, + K: 'static, +{ + // Wiped when this returns, as in `run_target`. + let plaintext = zeroize::Zeroizing::new(S::from_frame(name, &plaintext)?); + Ok(keyset + .encrypt_as::(&plaintext, column) + .try_map(|value| { + let query: Q = value.into(); + serde_json::to_vec(&query).map_err(|error| Error::Other(Box::new(error))) + })) +} + /// Parse a stored EQL value as the target and describe opening it through /// the target's own `DecryptInto`, under the column the caller expects. pub(crate) fn open_target<'a, T, S, K>( @@ -648,14 +706,36 @@ mod tests { assert!(matches!(refuse_query("Nope"), TargetError::Unknown { .. })); } + #[test] + fn a_query_through_an_index_the_type_lacks_is_refused() { + let text_eq = target("TextEq").unwrap(); + assert!(carries(text_eq, &IndexSpec::Equality).is_ok()); + let error = carries(text_eq, &IndexSpec::Ope).unwrap_err(); + assert!( + matches!( + error, + TargetError::NoIndex { + name: "TextEq", + index: "ope" + } + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + "TextEq carries no ope index, so no query is answered through it" + ); + } + #[test] fn refusals_name_the_type_and_its_reason() { assert!(matches!(refuse("Nope"), TargetError::Unknown { name } if name == "Nope")); assert!(matches!( - refuse("TextOrdOre"), - TargetError::Unproducible { name: "TextOrdOre", reason } if reason.contains("block ORE") + refuse("DateOrdOre"), + TargetError::Unproducible { name: "DateOrdOre", reason } if reason.contains("no order term for a date, timestamp or decimal") )); assert!(producible("TextEq").is_ok()); + assert!(producible("TextOrdOre").is_ok()); } fn frame(value: T) -> ValueFrame { diff --git a/packages/eql/crates/eql-bindings/src/v4/answers.rs b/packages/eql/crates/eql-bindings/src/v4/answers.rs new file mode 100644 index 000000000..ac030736d --- /dev/null +++ b/packages/eql/crates/eql-bindings/src/v4/answers.rs @@ -0,0 +1,218 @@ +// @generated by eql-codegen from the eql-domains catalog — do not edit +//! How Stack Encrypt's typed query chain queries a producible EQL v4 type: per type that answers an operator, its `op::Answers` impls, its `op::LabelledTarget` impl and its query twin's assembly (`crate::encryption::query`). Generated from the catalog, by the rule the data path answers a field that names the type with. +use crate::encryption::query::{QueryTwin, TwinIndex}; +use crate::encryption::EqlMatch; +use crate::v4::terms::{BloomFilter, Hmac256}; +use crate::v4::SchemaVersion; +use crate::Identifier; +use stack_encrypt::op; +use stack_encrypt::Equality; +use stack_encrypt::Match; +/// The operators each EQL type answers in Stack Encrypt's typed +/// query chain (`op::Answers`), by the wire names +/// `stack_encrypt::dynamic::OpSpec` gives them, in catalog order: +/// one row per type that answers any. Generated from the rule the +/// impls below are, so the encryption tests can hold the impls to +/// the operators the data path answers for each type. Not part of +/// the supported API. +#[doc(hidden)] +pub const ANSWERS: &[(&str, &[&str])] = &[ + ("IntegerEq", &["eq"]), + ("SmallintEq", &["eq"]), + ("BigintEq", &["eq"]), + ("DateEq", &["eq"]), + ("TimestampEq", &["eq"]), + ("NumericEq", &["eq"]), + ("TextEq", &["eq"]), + ("TextMatch", &["matches"]), + ("RealEq", &["eq"]), + ("DoubleEq", &["eq"]), +]; +/// `IntegerEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::integer::IntegerEq {} +/// `IntegerEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::integer::IntegerEqQuery { + type Plaintext = i32; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `IntegerEqQuery` is exact: the query term of `IntegerEq`'s `op::Eq`. +impl op::QueryTerm for super::integer::IntegerEqQuery {} +/// `IntegerEq` answers `op::Eq` through its equality index. The query term is `IntegerEqQuery`, bound as `eql_v4.query_integer_eq`. +impl op::Answers for super::integer::IntegerEq { + type Via = TwinIndex; +} +/// `SmallintEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::smallint::SmallintEq {} +/// `SmallintEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::smallint::SmallintEqQuery { + type Plaintext = i16; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `SmallintEqQuery` is exact: the query term of `SmallintEq`'s `op::Eq`. +impl op::QueryTerm for super::smallint::SmallintEqQuery {} +/// `SmallintEq` answers `op::Eq` through its equality index. The query term is `SmallintEqQuery`, bound as `eql_v4.query_smallint_eq`. +impl op::Answers for super::smallint::SmallintEq { + type Via = TwinIndex; +} +/// `BigintEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::bigint::BigintEq {} +/// `BigintEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::bigint::BigintEqQuery { + type Plaintext = i64; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `BigintEqQuery` is exact: the query term of `BigintEq`'s `op::Eq`. +impl op::QueryTerm for super::bigint::BigintEqQuery {} +/// `BigintEq` answers `op::Eq` through its equality index. The query term is `BigintEqQuery`, bound as `eql_v4.query_bigint_eq`. +impl op::Answers for super::bigint::BigintEq { + type Via = TwinIndex; +} +/// `DateEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::date::DateEq {} +/// `DateEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::date::DateEqQuery { + type Plaintext = crate::v4::plaintext::Date; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `DateEqQuery` is exact: the query term of `DateEq`'s `op::Eq`. +impl op::QueryTerm for super::date::DateEqQuery {} +/// `DateEq` answers `op::Eq` through its equality index. The query term is `DateEqQuery`, bound as `eql_v4.query_date_eq`. +impl op::Answers for super::date::DateEq { + type Via = TwinIndex; +} +/// `TimestampEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::timestamp::TimestampEq {} +/// `TimestampEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::timestamp::TimestampEqQuery { + type Plaintext = crate::v4::plaintext::Timestamp; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `TimestampEqQuery` is exact: the query term of `TimestampEq`'s `op::Eq`. +impl op::QueryTerm for super::timestamp::TimestampEqQuery {} +/// `TimestampEq` answers `op::Eq` through its equality index. The query term is `TimestampEqQuery`, bound as `eql_v4.query_timestamp_eq`. +impl op::Answers for super::timestamp::TimestampEq { + type Via = TwinIndex; +} +/// `NumericEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::numeric::NumericEq {} +/// `NumericEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::numeric::NumericEqQuery { + type Plaintext = crate::v4::plaintext::Decimal; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `NumericEqQuery` is exact: the query term of `NumericEq`'s `op::Eq`. +impl op::QueryTerm for super::numeric::NumericEqQuery {} +/// `NumericEq` answers `op::Eq` through its equality index. The query term is `NumericEqQuery`, bound as `eql_v4.query_numeric_eq`. +impl op::Answers for super::numeric::NumericEq { + type Via = TwinIndex; +} +/// `TextEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::text::TextEq {} +/// `TextEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::text::TextEqQuery { + type Plaintext = String; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `TextEqQuery` is exact: the query term of `TextEq`'s `op::Eq`. +impl op::QueryTerm for super::text::TextEqQuery {} +/// `TextEq` answers `op::Eq` through its equality index. The query term is `TextEqQuery`, bound as `eql_v4.query_text_eq`. +impl op::Answers for super::text::TextEq { + type Via = TwinIndex; +} +/// `TextMatch` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::text::TextMatch {} +/// `TextMatchQuery` from the column it queries and its `bf` term, at version 4. +impl QueryTwin for super::text::TextMatchQuery { + type Plaintext = String; + fn assemble(i: Identifier, bf: BloomFilter) -> Self { + Self { + v: SchemaVersion::default(), + i, + bf, + } + } +} +/// `TextMatch` answers `op::Matches` through its match index, under `EqlMatch`. The query term is a `MatchTwin`: the twin, bound as `eql_v4.query_text_match`, and the tokens that verify a row. +impl op::Answers for super::text::TextMatch { + type Via = TwinIndex, super::text::TextMatchQuery>; +} +/// `RealEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::real::RealEq {} +/// `RealEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::real::RealEqQuery { + type Plaintext = f32; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `RealEqQuery` is exact: the query term of `RealEq`'s `op::Eq`. +impl op::QueryTerm for super::real::RealEqQuery {} +/// `RealEq` answers `op::Eq` through its equality index. The query term is `RealEqQuery`, bound as `eql_v4.query_real_eq`. +impl op::Answers for super::real::RealEq { + type Via = TwinIndex; +} +/// `DoubleEq` is sealed under the column a two-segment label names, so its plan is queried under the plan's label. +impl op::LabelledTarget for super::double::DoubleEq {} +/// `DoubleEqQuery` from the column it queries and its `hm` term, at version 4. +impl QueryTwin for super::double::DoubleEqQuery { + type Plaintext = f64; + fn assemble(i: Identifier, hm: Hmac256) -> Self { + Self { + v: SchemaVersion::default(), + i, + hm, + } + } +} +/// `DoubleEqQuery` is exact: the query term of `DoubleEq`'s `op::Eq`. +impl op::QueryTerm for super::double::DoubleEqQuery {} +/// `DoubleEq` answers `op::Eq` through its equality index. The query term is `DoubleEqQuery`, bound as `eql_v4.query_double_eq`. +impl op::Answers for super::double::DoubleEq { + type Via = TwinIndex; +} diff --git a/packages/eql/crates/eql-bindings/src/v4/bigint.rs b/packages/eql/crates/eql-bindings/src/v4/bigint.rs index 359f1fff1..740d6c88d 100644 --- a/packages/eql/crates/eql-bindings/src/v4/bigint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/bigint.rs @@ -113,10 +113,18 @@ impl DomainType for BigintEq { doc, doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = i64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct BigintOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for BigintEqQuery { doc, doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`i64`](i64). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = i64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct BigintOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/double.rs b/packages/eql/crates/eql-bindings/src/v4/double.rs index c4aa41357..f2e00f4d9 100644 --- a/packages/eql/crates/eql-bindings/src/v4/double.rs +++ b/packages/eql/crates/eql-bindings/src/v4/double.rs @@ -113,10 +113,18 @@ impl DomainType for DoubleEq { doc, doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = f64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DoubleOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for DoubleEqQuery { doc, doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`f64`](f64). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = f64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DoubleOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/integer.rs b/packages/eql/crates/eql-bindings/src/v4/integer.rs index c0f661fe9..8fcb2c945 100644 --- a/packages/eql/crates/eql-bindings/src/v4/integer.rs +++ b/packages/eql/crates/eql-bindings/src/v4/integer.rs @@ -113,10 +113,18 @@ impl DomainType for IntegerEq { doc, doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = i32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct IntegerOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for IntegerEqQuery { doc, doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`i32`](i32). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = i32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct IntegerOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs new file mode 100644 index 000000000..f0d7576d5 --- /dev/null +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -0,0 +1,393 @@ +//! The `json_tree` family: an encrypted JSON document stored as one entry +//! per node, with node-local ciphertexts, a token chain and exact +//! containment. HAND-WRITTEN, like the v3 SteVec structs and for the same +//! reason: the bodies are not derivable from the catalog. The SQL is +//! `src/v3/json_tree/`, confined to EQL v4; the family is not in the +//! catalog yet, so it is not in [`super::all`] nor in the target table. +//! +//! Stack Encrypt's [`Json`](stack_encrypt::target::Json) index produces the +//! document ([`stack_encrypt::json::JsonDocument`], whose wire form is the +//! `h` and `sv` here) and the query operands; these types are the SQL +//! contract around them, with the `v` and `i` envelope, and the +//! [`EncryptFrom`] / [`DecryptInto`] impls that let a Rust program name +//! [`JsonTreeDocument`] as a target: `encrypt_as::` under +//! an [`Identifier`]. A query derives through the engine's own document +//! target under the same context as a label, `Plan::value::() +//! .context("users/profile").encrypt_into::()`, and converts +//! into the operand types here; the identifier and the two-segment label +//! are one context, so the selectors agree (see [`crate::encryption`]). The +//! design is `docs/plans/2026-10-17-json-tree-index.md` at +//! the repository root. + +use schemars::{schema_for, JsonSchema, Schema}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use stack_encrypt::json::{JsonDocument, JsonLeafQuery, JsonNode, JsonQuery, JsonSubtree}; +use stack_encrypt::target::{AeadContext, ExpectedContext, IndexSpec}; +use stack_encrypt::{ + DecryptField, DecryptInto, Decryptable, Decryption, EncryptFrom, Encryption, Error, NonEmpty, +}; +use ts_rs::TS; + +use crate::v3::terms::{EntryCiphertext, KeyHeader, OpeCllw, Selector}; +use crate::v4::{DomainType, SchemaVersion}; +use crate::Identifier; + +/// A sixteen-byte chain tag, hex: a parent tag `p`, a wrapped token `w`, +/// a group tag `n` or `g`. Per document, salted by the document's `iv`, so +/// unlinkable across rows at rest. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)] +pub struct JsonTreeTag(pub String); + +/// `public.eql_v4_json_tree`: a JSON tree document, `{v, i, h, sv: +/// [entry]}`. Strict. `h` is the document's key header, stored once (every +/// entry is sealed under the document's one data key); `sv` holds one +/// entry per node in depth-first order, the root first. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeDocument { + pub v: SchemaVersion, + pub i: Identifier, + pub h: KeyHeader, + pub sv: Vec, +} + +/// `public.eql_v4_json_tree_entry`: one node. `s` and `vs` are the path +/// and value selectors, `c` the node's own ciphertext; `o`, `p` and `w` +/// the chain fields below the heads; `n` and `g` the group tags of +/// container array elements and what sits inside them; `op` the leaf +/// ordering term. `h`, `j`, `d`, `v` and `i` are what the extractors graft +/// onto an entry they return, so it opens on its own. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeEntry { + pub s: Selector, + pub vs: Selector, + pub c: EntryCiphertext, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub o: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub w: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub n: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub g: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub op: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub h: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub j: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub d: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub v: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub i: Option, +} + +/// One selector pair of a containment needle. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeQueryPair { + pub s: Selector, + pub vs: Selector, +} + +/// One group of a containment needle: the pairs that sit directly in it, +/// and one nested group per array element that is a container. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeQueryGroup { + pub sv: Vec, + pub el: Vec, +} + +/// `eql_v4.query_json_tree`: a containment needle, the element-nested +/// projection of the needle document's selectors, with the alternative +/// projection a root scalar carries. Strict. What +/// [`stack_encrypt::json::JsonQuery`] serializes to; [`TryFrom`] converts +/// one. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeQuery { + pub sv: Vec, + pub el: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub alt: Option, +} + +/// `eql_v4.query_json_tree_leaf`: an ordering operand at a path, the path's +/// selector and the leaf ordering term. What +/// [`stack_encrypt::json::JsonLeafQuery`] serializes to. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeLeafQuery { + pub s: Selector, + pub op: OpeCllw, +} + +/// The rows a subtree walk returned (`eql_v4.json_tree_subtree` or +/// `json_tree_subtree_at`), collected: opens to the value under the +/// walked node, or to the list of complete subtrees the rows hold. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct JsonTreeSubtree(pub Vec); + +macro_rules! json_tree_domain_type { + ($ty:ident, $sql:literal) => { + impl DomainType for $ty { + fn sql_domain_static() -> &'static str { + $sql + } + fn sql_domain(&self) -> &'static str { + Self::sql_domain_static() + } + // `term_json_keys` keeps the trait default (`None`): the terms + // live per entry, not as flat payload keys. + fn parse_value(&self, value: &serde_json::Value) -> Result<(), serde_json::Error> { + <$ty as Deserialize>::deserialize(value).map(|_| ()) + } + fn schema(&self) -> Schema { + schema_for!($ty) + } + fn schema_id_base_static() -> &'static str { + super::SCHEMA_ID_BASE + } + fn schema_id(&self) -> String { + format!("{}{}.json", super::SCHEMA_ID_BASE, self.domain()) + } + } + }; +} + +json_tree_domain_type!(JsonTreeDocument, "public.eql_v4_json_tree"); +json_tree_domain_type!(JsonTreeEntry, "public.eql_v4_json_tree_entry"); +json_tree_domain_type!(JsonTreeQuery, "eql_v4.query_json_tree"); +json_tree_domain_type!(JsonTreeLeafQuery, "eql_v4.query_json_tree_leaf"); + +// ============================================================================= +// Conversions to and from the engine's types, through their one wire form +// ============================================================================= + +fn codec(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::Other(Box::new(error)) +} + +/// Re-read one serializable value as another type sharing its wire form. +fn transcode(value: &T) -> Result { + serde_json::from_value(serde_json::to_value(value).map_err(codec)?).map_err(codec) +} + +impl JsonTreeDocument { + /// The envelope around an engine document: `v = 4`, the column, and + /// the document's header and entries. + fn envelope(i: Identifier, document: &JsonDocument) -> Result { + let mut value = serde_json::to_value(document).map_err(codec)?; + let object = value + .as_object_mut() + .ok_or_else(|| Error::Other("a JSON document serializes to an object".into()))?; + let _ = object.insert("v".into(), serde_json::json!(4)); + let _ = object.insert("i".into(), serde_json::to_value(i).map_err(codec)?); + serde_json::from_value(value).map_err(codec) + } + + /// The engine's document: the header and entries without the envelope. + fn document(&self) -> Result { + serde_json::from_value(serde_json::json!({ "h": self.h, "sv": self.sv })).map_err(codec) + } +} + +impl TryFrom for JsonTreeQuery { + type Error = Error; + fn try_from(query: JsonQuery) -> Result { + transcode(&query) + } +} + +impl TryFrom<&JsonQuery> for JsonTreeQuery { + type Error = Error; + fn try_from(query: &JsonQuery) -> Result { + transcode(query) + } +} + +impl TryFrom<&JsonLeafQuery> for JsonTreeLeafQuery { + type Error = Error; + fn try_from(query: &JsonLeafQuery) -> Result { + transcode(query) + } +} + +impl JsonTreeEntry { + /// The entry as an extracted node the engine opens: it must carry the + /// grafted `h` and `j`, which every extractor adds. + fn node(&self) -> Result { + if self.h.is_none() || self.j.is_none() { + return Err(Error::Other( + "a JSON tree entry opens only as an extractor returned it, with `h` and `j`".into(), + )); + } + transcode(self) + } +} + +// ============================================================================= +// Stack Encrypt targets +// ============================================================================= + +/// A document seals as the JSON index lays it out, under the column it is +/// stored in: the [`Identifier`] is the context the data key is minted +/// under and every selector derives under, and is written to `i`. +impl EncryptFrom for JsonTreeDocument { + type Context = NonEmpty; + fn encryption<'s, K: 'static>() -> Encryption<'s, Value, Self, K, Self::Context> { + >::encryption() + .accepting::>() + .map_with_context(|document, i: NonEmpty| (i, document)) + .try_map(|(i, document)| Self::envelope(i.into_inner(), &document)) + } + fn indexes() -> Vec { + >::indexes() + } +} + +/// Opens to the plaintext document, under the stored identifier, which an +/// [`ExpectedContext`] may require to be the column the caller expects. +impl DecryptInto for JsonTreeDocument { + type Context = ExpectedContext; + fn decryption(self, context: Self::Context) -> Decryption { + let column = match context.validate(self.i.clone()) { + Ok(column) => column, + Err(error) => return Decryption::failed(error), + }; + match self.document() { + Ok(document) => document.decryption(AeadContext::from(column)), + Err(error) => Decryption::failed(error), + } + } +} + +impl Decryptable for JsonTreeDocument { + const DECRYPTABLE: bool = true; +} +impl DecryptField for JsonTreeDocument +where + Self: DecryptInto

, + Ctx: Into<>::Context>, +{ + fn decryption_field(self, context: Ctx) -> Option> { + Some(self.decryption(context.into())) + } +} + +/// An extracted entry opens to the scalar it holds, or to the whole +/// document for a sealed root, under the column it was extracted from. A +/// container node holds no value of its own; its subtree opens through +/// [`JsonTreeSubtree`]. +impl DecryptInto for JsonTreeEntry { + type Context = NonEmpty; + fn decryption(self, context: Self::Context) -> Decryption { + match self.node() { + Ok(node) => node.decryption(AeadContext::from(context)), + Err(error) => Decryption::failed(error), + } + } +} + +/// The rows of `json_tree_subtree(doc, token)` open to the value under the +/// walked node. +impl DecryptInto for JsonTreeSubtree { + type Context = NonEmpty; + fn decryption(self, context: Self::Context) -> Decryption { + match self.subtree() { + Ok(subtree) => subtree.decryption(AeadContext::from(context)), + Err(error) => Decryption::failed(error), + } + } +} + +/// The rows of a walk open to the complete subtrees they hold, in order: +/// one per node whose selector `json_tree_subtree_at` matched. +impl DecryptInto> for JsonTreeSubtree { + type Context = NonEmpty; + fn decryption(self, context: Self::Context) -> Decryption, K> { + match self.subtree() { + Ok(subtree) => subtree.decryption(AeadContext::from(context)), + Err(error) => Decryption::failed(error), + } + } +} + +impl JsonTreeSubtree { + fn subtree(&self) -> Result { + Ok(JsonSubtree::new( + self.0 + .iter() + .map(JsonTreeEntry::node) + .collect::>()?, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_domains_are_the_v4_json_tree_family() { + assert_eq!( + JsonTreeDocument::sql_domain_static(), + "public.eql_v4_json_tree" + ); + assert_eq!( + JsonTreeEntry::sql_domain_static(), + "public.eql_v4_json_tree_entry" + ); + assert_eq!(JsonTreeQuery::sql_domain_static(), "eql_v4.query_json_tree"); + assert_eq!( + JsonTreeLeafQuery::sql_domain_static(), + "eql_v4.query_json_tree_leaf" + ); + } + + #[test] + fn a_query_needs_sv_and_el_and_takes_an_alt() { + let query: JsonTreeQuery = serde_json::from_value(serde_json::json!({ + "sv": [{"s": "aa", "vs": "bb"}], + "el": [{"sv": [], "el": []}], + "alt": {"sv": [{"s": "cc", "vs": "dd"}], "el": []}, + })) + .unwrap(); + assert_eq!(query.el.len(), 1); + assert!(query.alt.is_some()); + assert!(serde_json::from_value::(serde_json::json!({"sv": []})).is_err()); + assert!(serde_json::from_value::(serde_json::json!({ + "sv": [], "el": [], "grp": [] + })) + .is_err()); + } + + #[test] + fn an_entry_without_the_grafted_fields_does_not_open() { + let entry: JsonTreeEntry = serde_json::from_value(serde_json::json!({ + "s": "00", "vs": "00", "c": "AA==" + })) + .unwrap(); + assert!(entry.node().is_err()); + } +} diff --git a/packages/eql/crates/eql-bindings/src/v4/mod.rs b/packages/eql/crates/eql-bindings/src/v4/mod.rs index 45a6242cc..02e360f83 100644 --- a/packages/eql/crates/eql-bindings/src/v4/mod.rs +++ b/packages/eql/crates/eql-bindings/src/v4/mod.rs @@ -20,12 +20,22 @@ //! operand of `eql_v4.between(column, range)`. EQL v3 has no range domains. //! //! v4 has no SteVec bindings, payload enums, TypeScript export or -//! published JSON Schema yet; those follow in `@cipherstash/eql` 4.0. The +//! published JSON Schema yet; those follow in `@cipherstash/eql` 4.0. It +//! does have the hand-written [`json_tree`] family, the JSON tree document +//! Stack Encrypt's JSON index writes, which is a new domain beside the +//! SteVec surface and not a port of it. The //! term newtypes and the [`DomainType`] trait are shared with v3: a v4 //! `Ciphertext` or `Hmac256` is v3's type, so a change to a term's validation -//! in `v3::terms` changes v4 too. What separates the two on the wire is the -//! envelope's [`SchemaVersion`], which refuses the other generation's `v`. +//! in `v3::terms` changes v4 too. The one v4 term v3 has no use for is +//! [`terms::SearchBloomFilter`], the stored filter of a search domain. What +//! separates the two on the wire is the envelope's [`SchemaVersion`], which +//! refuses the other generation's `v`. +/// Generated: the `op::Answers` impls through which Stack Encrypt's typed +/// query chain queries a producible type, each type's +/// `op::LabelledTarget` impl and its query twin's assembly +/// ([`crate::encryption::query`]). Needs only the `stack-encrypt` feature. +pub mod answers; pub mod bigint; pub mod boolean; pub mod date; @@ -35,6 +45,10 @@ pub mod inventory; /// Generated bindings for the `json` family's bare scalar storage domain /// (`public.eql_v4_json`, the `Json` struct). pub mod json; +/// The hand-written `json_tree` family: the JSON tree document Stack +/// Encrypt's JSON index writes, its entry, and its two query operands. +/// EQL v4 only, and not in the catalog yet, so not in [`all`]. +pub mod json_tree; pub mod numeric; // The Rust plaintexts of the `date`, `timestamp` and `numeric` families. A // plain comment: an outer doc comment here would resolve the module's own @@ -53,9 +67,58 @@ pub mod text; pub mod timestamp; pub use crate::v3::domain_type::{DomainType, SQL_SCHEMA}; -pub use crate::v3::terms; pub use inventory::{all, all_query, all_range_query}; +/// The wire-field newtypes of the v4 payloads: v3's, shared, and +/// [`SearchBloomFilter`](terms::SearchBloomFilter). +pub mod terms { + pub use crate::v3::terms::*; + + use schemars::JsonSchema; + use serde::{Deserialize, Serialize}; + use ts_rs::TS; + + /// The `bf` term of a stored search value (`TextSearch`, + /// `TextSearchOre`): a [`BloomFilter`] that may be empty. + /// + /// Text too short to cut a 3-gram (empty, separator-only, or shorter than + /// three characters) has no match tokens. A search column stores it with + /// the empty filter, Stack Encrypt's answer for a stored value, so `=` + /// and `<` still find it through its `hm` and `op`, and no `@@` query + /// does: EQL's `matches` finds a stored filter only when it contains the + /// query's, and an EQL v4 search query's filter is never empty (the + /// match projection refuses such text, and the SQL refuses `"bf": []`). + /// + /// [`BloomFilter`], the filter of `TextMatch` and of every query, still + /// refuses such text: `TextMatch` has no other term to find it by. The + /// JSON is [`BloomFilter`]'s, and so is the JSON Schema; only what Stack + /// Encrypt derives into it differs. + #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS)] + #[serde(transparent)] + pub struct SearchBloomFilter(pub BloomFilter); + + /// [`BloomFilter`]'s schema, under its name: the wire format is one. + impl JsonSchema for SearchBloomFilter { + fn schema_name() -> std::borrow::Cow<'static, str> { + BloomFilter::schema_name() + } + + fn schema_id() -> std::borrow::Cow<'static, str> { + BloomFilter::schema_id() + } + + fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + BloomFilter::json_schema(generator) + } + } + + impl From for BloomFilter { + fn from(filter: SearchBloomFilter) -> Self { + filter.0 + } + } +} + /// The PostgreSQL typname prefix every `public` v4 domain carries. pub const PUBLIC_TYPNAME_PREFIX: &str = "eql_v4_"; diff --git a/packages/eql/crates/eql-bindings/src/v4/real.rs b/packages/eql/crates/eql-bindings/src/v4/real.rs index 3def16c0e..945374401 100644 --- a/packages/eql/crates/eql-bindings/src/v4/real.rs +++ b/packages/eql/crates/eql-bindings/src/v4/real.rs @@ -113,10 +113,18 @@ impl DomainType for RealEq { doc, doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = f32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RealOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for RealEqQuery { doc, doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`f32`](f32). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = f32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RealOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/smallint.rs b/packages/eql/crates/eql-bindings/src/v4/smallint.rs index a3f7eb236..503f7bf7b 100644 --- a/packages/eql/crates/eql-bindings/src/v4/smallint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/smallint.rs @@ -113,10 +113,18 @@ impl DomainType for SmallintEq { doc, doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = i16)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SmallintOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for SmallintEqQuery { doc, doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`i16`](i16). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = i16)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SmallintOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index 67d8e09e0..feddec846 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -1,13 +1,15 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The EQL types a Stack Encrypt data plan may name as a field target — every stored EQL v4 domain type in eql-domains::CATALOG order, as data a guest serializes (`TARGETS`), with the by-name dispatch that runs a producible type's own Rust plan. Generated from the catalog; the descriptor type, the errors and the public entry points stay hand-written in `crate::encryption::targets`, which documents the wire format. use crate::encryption::targets::{ - open_target, refuse, refuse_query, run_target, Opener, Target, TargetError, + no_index, open_target, refuse, refuse_query, run_projected, run_target, Opener, Target, + TargetError, }; use crate::v4::plaintext::Date; use crate::v4::plaintext::Decimal; use crate::v4::plaintext::Timestamp; use crate::Identifier; use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::target::IndexSpec; use stack_encrypt::{KeysetCipher, NonEmpty, Pending}; /// Every EQL type a plan may name as a target, in /// `eql-domains::CATALOG` order: one per stored domain (every flat @@ -44,13 +46,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("int32"), sql_domain: "public.eql_v4_integer_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("IntegerOrdOreQuery"), query_sql_domain: Some("eql_v4.query_integer_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "IntegerOrd", @@ -106,13 +106,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("int16"), sql_domain: "public.eql_v4_smallint_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("SmallintOrdOreQuery"), query_sql_domain: Some("eql_v4.query_smallint_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "SmallintOrd", @@ -168,13 +166,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("int64"), sql_domain: "public.eql_v4_bigint_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("BigintOrdOreQuery"), query_sql_domain: Some("eql_v4.query_bigint_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "BigintOrd", @@ -230,7 +226,7 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("date"), sql_domain: "public.eql_v4_date_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("DateOrdOreQuery"), query_sql_domain: Some("eql_v4.query_date_ord_ore"), producible: false, @@ -296,7 +292,7 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("timestamp"), sql_domain: "public.eql_v4_timestamp_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("TimestampOrdOreQuery"), query_sql_domain: Some("eql_v4.query_timestamp_ord_ore"), producible: false, @@ -362,7 +358,7 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("decimal"), sql_domain: "public.eql_v4_numeric_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("NumericOrdOreQuery"), query_sql_domain: Some("eql_v4.query_numeric_ord_ore"), producible: false, @@ -440,13 +436,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("string"), sql_domain: "public.eql_v4_text_ord_ore", - indexes: &["eq", "ore"], + indexes: &["eq", "ore_block"], query: Some("TextOrdOreQuery"), query_sql_domain: Some("eql_v4.query_text_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "TextOrd", @@ -478,13 +472,11 @@ pub const TARGETS: &[Target] = &[ suffix: "SearchOre", plaintext: Some("string"), sql_domain: "public.eql_v4_text_search_ore", - indexes: &["eq", "ore", "match"], + indexes: &["eq", "ore_block", "match"], query: Some("TextSearchOreQuery"), query_sql_domain: Some("eql_v4.query_text_search_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "TextSearch", @@ -495,10 +487,8 @@ pub const TARGETS: &[Target] = &[ indexes: &["eq", "ope", "match"], query: Some("TextSearchQuery"), query_sql_domain: Some("eql_v4.query_text_search"), - producible: false, - reason: Some( - "every query of this type carries a match term, which text shorter than one 3-gram cannot make, so `=` or `<` on short text would be refused; how a query that is not a match leaves it out is not decided (#1131)", - ), + producible: true, + reason: None, }, Target { name: "Boolean", @@ -542,13 +532,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("float32"), sql_domain: "public.eql_v4_real_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("RealOrdOreQuery"), query_sql_domain: Some("eql_v4.query_real_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "RealOrd", @@ -604,13 +592,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("float64"), sql_domain: "public.eql_v4_double_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("DoubleOrdOreQuery"), query_sql_domain: Some("eql_v4.query_double_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "DoubleOrd", @@ -679,6 +665,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "IntegerEq" => { run_target::("IntegerEq", keyset, column, plaintext) } + "IntegerOrdOre" => run_target::( + "IntegerOrdOre", + keyset, + column, + plaintext, + ), "IntegerOrd" => run_target::( "IntegerOrd", keyset, @@ -700,6 +692,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), + "SmallintOrdOre" => run_target::( + "SmallintOrdOre", + keyset, + column, + plaintext, + ), "SmallintOrd" => run_target::( "SmallintOrd", keyset, @@ -718,6 +716,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "BigintEq" => { run_target::("BigintEq", keyset, column, plaintext) } + "BigintOrdOre" => run_target::( + "BigintOrdOre", + keyset, + column, + plaintext, + ), "BigintOrd" => { run_target::("BigintOrd", keyset, column, plaintext) } @@ -757,6 +761,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "TextMatch" => { run_target::("TextMatch", keyset, column, plaintext) } + "TextOrdOre" => run_target::( + "TextOrdOre", + keyset, + column, + plaintext, + ), "TextOrd" => { run_target::("TextOrd", keyset, column, plaintext) } @@ -766,11 +776,26 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), + "TextSearchOre" => run_target::( + "TextSearchOre", + keyset, + column, + plaintext, + ), + "TextSearch" => run_target::( + "TextSearch", + keyset, + column, + plaintext, + ), "Boolean" => { run_target::("Boolean", keyset, column, plaintext) } "Real" => run_target::("Real", keyset, column, plaintext), "RealEq" => run_target::("RealEq", keyset, column, plaintext), + "RealOrdOre" => { + run_target::("RealOrdOre", keyset, column, plaintext) + } "RealOrd" => { run_target::("RealOrd", keyset, column, plaintext) } @@ -783,6 +808,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "DoubleEq" => { run_target::("DoubleEq", keyset, column, plaintext) } + "DoubleOrdOre" => run_target::( + "DoubleOrdOre", + keyset, + column, + plaintext, + ), "DoubleOrd" => { run_target::("DoubleOrd", keyset, column, plaintext) } @@ -810,6 +841,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "IntegerEq" => { open_target::("IntegerEq", opener, column, stored) } + "IntegerOrdOre" => open_target::( + "IntegerOrdOre", + opener, + column, + stored, + ), "IntegerOrd" => { open_target::("IntegerOrd", opener, column, stored) } @@ -825,6 +862,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "SmallintEq" => { open_target::("SmallintEq", opener, column, stored) } + "SmallintOrdOre" => open_target::( + "SmallintOrdOre", + opener, + column, + stored, + ), "SmallintOrd" => open_target::( "SmallintOrd", opener, @@ -841,6 +884,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "BigintEq" => { open_target::("BigintEq", opener, column, stored) } + "BigintOrdOre" => open_target::( + "BigintOrdOre", + opener, + column, + stored, + ), "BigintOrd" => { open_target::("BigintOrd", opener, column, stored) } @@ -878,17 +927,32 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "TextMatch" => { open_target::("TextMatch", opener, column, stored) } + "TextOrdOre" => { + open_target::("TextOrdOre", opener, column, stored) + } "TextOrd" => { open_target::("TextOrd", opener, column, stored) } "TextOrdOpe" => { open_target::("TextOrdOpe", opener, column, stored) } + "TextSearchOre" => open_target::( + "TextSearchOre", + opener, + column, + stored, + ), + "TextSearch" => { + open_target::("TextSearch", opener, column, stored) + } "Boolean" => { open_target::("Boolean", opener, column, stored) } "Real" => open_target::("Real", opener, column, stored), "RealEq" => open_target::("RealEq", opener, column, stored), + "RealOrdOre" => { + open_target::("RealOrdOre", opener, column, stored) + } "RealOrd" => open_target::("RealOrd", opener, column, stored), "RealOrdOpe" => { open_target::("RealOrdOpe", opener, column, stored) @@ -897,6 +961,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "DoubleEq" => { open_target::("DoubleEq", opener, column, stored) } + "DoubleOrdOre" => open_target::( + "DoubleOrdOre", + opener, + column, + stored, + ), "DoubleOrd" => { open_target::("DoubleOrd", opener, column, stored) } @@ -912,8 +982,11 @@ pub(crate) fn decrypt_named<'a, K: 'static>( /// Run the named type's query twin for one plaintext, or refuse the /// name: as `encrypt_named` does, or as answering no query /// (`TargetError::NoQuery`) for a producible type with no twin. +/// `index` answers the query's operator; a twin whose terms are +/// optional carries that index's term alone. pub(crate) fn query_named<'a, K: 'static>( name: &str, + index: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, column: NonEmpty, plaintext: ValueFrame, @@ -925,6 +998,12 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "IntegerOrdOre" => run_target::( + "IntegerOrdOre", + keyset, + column, + plaintext, + ), "IntegerOrd" => run_target::( "IntegerOrd", keyset, @@ -943,6 +1022,12 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "SmallintOrdOre" => run_target::( + "SmallintOrdOre", + keyset, + column, + plaintext, + ), "SmallintOrd" => run_target::( "SmallintOrd", keyset, @@ -958,6 +1043,12 @@ pub(crate) fn query_named<'a, K: 'static>( "BigintEq" => run_target::( "BigintEq", keyset, column, plaintext, ), + "BigintOrdOre" => run_target::( + "BigintOrdOre", + keyset, + column, + plaintext, + ), "BigintOrd" => run_target::( "BigintOrd", keyset, @@ -994,6 +1085,12 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "TextOrdOre" => run_target::( + "TextOrdOre", + keyset, + column, + plaintext, + ), "TextOrd" => { run_target::("TextOrd", keyset, column, plaintext) } @@ -1003,9 +1100,57 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "TextSearchOre" => match index.key() { + "eq" => run_projected::< + super::text::TextSearchOreEqualityQuery, + super::text::TextSearchOreQuery, + String, + K, + >("TextSearchOre", keyset, column, plaintext), + "ore_block" => run_projected::< + super::text::TextSearchOreOrderQuery, + super::text::TextSearchOreQuery, + String, + K, + >("TextSearchOre", keyset, column, plaintext), + "match" => run_projected::< + super::text::TextSearchOreMatchQuery, + super::text::TextSearchOreQuery, + String, + K, + >("TextSearchOre", keyset, column, plaintext), + _ => Err(no_index("TextSearchOre", index)), + }, + "TextSearch" => match index.key() { + "eq" => run_projected::< + super::text::TextSearchEqualityQuery, + super::text::TextSearchQuery, + String, + K, + >("TextSearch", keyset, column, plaintext), + "ope" => run_projected::< + super::text::TextSearchOrderQuery, + super::text::TextSearchQuery, + String, + K, + >("TextSearch", keyset, column, plaintext), + "match" => run_projected::< + super::text::TextSearchMatchQuery, + super::text::TextSearchQuery, + String, + K, + >("TextSearch", keyset, column, plaintext), + _ => Err(no_index("TextSearch", index)), + }, "RealEq" => { run_target::("RealEq", keyset, column, plaintext) } + "RealOrdOre" => run_target::( + "RealOrdOre", + keyset, + column, + plaintext, + ), "RealOrd" => { run_target::("RealOrd", keyset, column, plaintext) } @@ -1018,6 +1163,12 @@ pub(crate) fn query_named<'a, K: 'static>( "DoubleEq" => run_target::( "DoubleEq", keyset, column, plaintext, ), + "DoubleOrdOre" => run_target::( + "DoubleOrdOre", + keyset, + column, + plaintext, + ), "DoubleOrd" => run_target::( "DoubleOrd", keyset, diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index fb7e5546a..af5544826 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -1,6 +1,6 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `text` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256, SearchBloomFilter}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; @@ -163,10 +163,18 @@ impl DomainType for TextMatch { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, @@ -308,15 +316,23 @@ impl DomainType for TextOrdOpe { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearchOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, pub ob: OreBlock256, - pub bf: BloomFilter, + pub bf: SearchBloomFilter, } impl DomainType for TextSearchOre { fn sql_domain_static() -> &'static str { @@ -352,15 +368,23 @@ impl DomainType for TextSearchOre { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearch { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, pub op: OpeCllw, - pub bf: BloomFilter, + pub bf: SearchBloomFilter, } impl DomainType for TextSearch { fn sql_domain_static() -> &'static str { @@ -494,10 +518,18 @@ impl DomainType for TextMatchQuery { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality and order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub hm: Hmac256, pub ob: OreBlock256, @@ -630,20 +662,25 @@ impl DomainType for TextOrdOpeQuery { } /// `eql_v4.query_text_search_ore` — search domain query operand. /// -/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i` `hm` `ob` `bf`. +/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i`, and at least one of `hm` `ob` `bf`: a query carries the term its operator reads. #[cfg_attr(doc, doc = "")] #[cfg_attr( doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`String`](String), through one projection per index — [`TextSearchOreEqualityQuery`], [`TextSearchOreOrderQuery`], [`TextSearchOreMatchQuery`] — each `EncryptFrom` and `Into`: a query carries the term of the index that answers its operator. Query operands do not decrypt. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearchOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub hm: Hmac256, - pub ob: OreBlock256, - pub bf: BloomFilter, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ob: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bf: Option, } impl DomainType for TextSearchOreQuery { fn sql_domain_static() -> &'static str { @@ -671,22 +708,102 @@ impl DomainType for TextSearchOreQuery { format!("{}{}.json", super::SCHEMA_ID_BASE, self.domain()) } } +/// The `eql_v4.query_text_search_ore` query operand for `=` `<>`: the `hm` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchOreQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOreEqualityQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub hm: Hmac256, +} +impl From for TextSearchOreQuery { + fn from(query: TextSearchOreEqualityQuery) -> Self { + TextSearchOreQuery { + v: query.v, + i: query.i, + hm: Some(query.hm), + ob: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search_ore` query operand for `<` `<=` `>` `>=`: the `ob` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchOreQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOreOrderQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub ob: OreBlock256, +} +impl From for TextSearchOreQuery { + fn from(query: TextSearchOreOrderQuery) -> Self { + TextSearchOreQuery { + v: query.v, + i: query.i, + ob: Some(query.ob), + hm: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search_ore` query operand for `@@`: the `bf` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchOreQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOreMatchQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub bf: BloomFilter, +} +impl From for TextSearchOreQuery { + fn from(query: TextSearchOreMatchQuery) -> Self { + TextSearchOreQuery { + v: query.v, + i: query.i, + bf: Some(query.bf), + hm: None, + ob: None, + } + } +} /// `eql_v4.query_text_search` — search domain query operand. /// -/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i` `hm` `op` `bf`. +/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i`, and at least one of `hm` `op` `bf`: a query carries the term its operator reads. #[cfg_attr(doc, doc = "")] #[cfg_attr( doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`String`](String), through one projection per index — [`TextSearchEqualityQuery`], [`TextSearchOrderQuery`], [`TextSearchMatchQuery`] — each `EncryptFrom` and `Into`: a query carries the term of the index that answers its operator. Query operands do not decrypt. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearchQuery { pub v: SchemaVersion, pub i: Identifier, - pub hm: Hmac256, - pub op: OpeCllw, - pub bf: BloomFilter, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub op: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bf: Option, } impl DomainType for TextSearchQuery { fn sql_domain_static() -> &'static str { @@ -714,6 +831,81 @@ impl DomainType for TextSearchQuery { format!("{}{}.json", super::SCHEMA_ID_BASE, self.domain()) } } +/// The `eql_v4.query_text_search` query operand for `=` `<>`: the `hm` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchEqualityQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub hm: Hmac256, +} +impl From for TextSearchQuery { + fn from(query: TextSearchEqualityQuery) -> Self { + TextSearchQuery { + v: query.v, + i: query.i, + hm: Some(query.hm), + op: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search` query operand for `<` `<=` `>` `>=`: the `op` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOrderQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub op: OpeCllw, +} +impl From for TextSearchQuery { + fn from(query: TextSearchOrderQuery) -> Self { + TextSearchQuery { + v: query.v, + i: query.i, + op: Some(query.op), + hm: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search` query operand for `@@`: the `bf` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchMatchQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub bf: BloomFilter, +} +impl From for TextSearchQuery { + fn from(query: TextSearchMatchQuery) -> Self { + TextSearchQuery { + v: query.v, + i: query.i, + bf: Some(query.bf), + hm: None, + op: None, + } + } +} /// `eql_v4.query_text_range_ore` — block ORE range query operand: both bounds of one `BETWEEN`. /// /// Function: `eql_v4.between(column, range)`, both bounds included. Columns: `public.eql_v4_text_ord_ore` `public.eql_v4_text_search_ore`. Required keys: `v` `i` `lo` `hi`. diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 42bf7f7c1..cb5011b29 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -14,6 +14,7 @@ use quote::{format_ident, quote}; use eql_domains::{Domain, DomainFamily, Term, CATALOG, ENVELOPE_KEYS, RANGE_BOUND_KEYS}; use crate::consts::RUST_GENERATED_MARKER; +use crate::context::v4_optional_query_terms; use crate::render::EqlVersion; use crate::writer::{ ensure_generated_paths_writable, normalized_set, remove_generated_orphans, @@ -198,7 +199,7 @@ fn render_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) -> Tok fields.extend(quote! { pub c: Ciphertext, }); for term in Term::payload_terms(domain.terms) { let fid = format_ident!("{}", term.json_key()); - let tid = format_ident!("{}", term.binding_newtype()); + let tid = format_ident!("{}", stored_term_newtype(eql, domain, term)); fields.extend(quote! { pub #fid: #tid, }); } @@ -276,13 +277,30 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) .map(|o| format!("`{o}`")) .collect::>() .join(" "); - let keys_str = ["v", "i"] - .into_iter() - .chain(Term::term_json_keys(domain.terms)) - .map(|k| format!("`{k}`")) - .collect::>() - .join(" "); - let detail = format!(" Operators: {ops_str}. Required keys: {keys_str}."); + let quoted = |keys: &mut dyn Iterator| { + keys.map(|k| format!("`{k}`")).collect::>().join(" ") + }; + // An EQL v4 query whose terms are optional carries the one its operator + // reads (ADR-0002, amended 2026-10-19): each term is an `Option`, absent + // from the JSON when `None`, and the SQL domain requires one of them. + let optional = eql == EqlVersion::V4 && v4_optional_query_terms(domain); + let detail = if optional { + format!( + " Operators: {ops_str}. Required keys: {}, and at least one of {}: a query carries \ + the term its operator reads.", + quoted(&mut ["v", "i"].into_iter()), + quoted(&mut Term::term_json_keys(domain.terms).into_iter()), + ) + } else { + format!( + " Operators: {ops_str}. Required keys: {}.", + quoted( + &mut ["v", "i"] + .into_iter() + .chain(Term::term_json_keys(domain.terms)) + ) + ) + }; // Envelope minus `c`: `v`/`i` only. Kept in lockstep with the storage // struct's envelope triple (see `envelope_fields_match_catalog_keys`). @@ -292,9 +310,17 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) for term in Term::payload_terms(domain.terms) { let fid = format_ident!("{}", term.json_key()); let tid = format_ident!("{}", term.binding_newtype()); - fields.extend(quote! { pub #fid: #tid, }); + fields.extend(if optional { + quote! { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub #fid: Option<#tid>, + } + } else { + quote! { pub #fid: #tid, } + }); } let term_keys = Term::term_json_keys(domain.terms); + let projections = render_query_projections(eql, family, domain); quote! { #[doc = #summary] @@ -330,6 +356,8 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) } #schema_id } + + #projections } } @@ -411,24 +439,164 @@ fn render_range_query_struct(family: &DomainFamily, term: Term) -> TokenStream { } } +/// The newtype of a stored struct's `term` field. The term's own +/// ([`Term::binding_newtype`]), except the match term of an EQL v4 search +/// domain (one whose query terms are optional, [`v4_optional_query_terms`]): +/// `SearchBloomFilter`, the same JSON as `BloomFilter`, into which Stack +/// Encrypt derives the empty filter for text too short for a 3-gram instead +/// of refusing it. Such a value is still found by `=` and `<` through its +/// other terms; a query, and `TextMatch`, which has no other term, keep +/// `BloomFilter` and keep refusing that text. +pub fn stored_term_newtype(eql: EqlVersion, domain: &Domain, term: Term) -> &'static str { + if eql == EqlVersion::V4 && term == Term::Bloom && v4_optional_query_terms(domain) { + "SearchBloomFilter" + } else { + term.binding_newtype() + } +} + +/// The term newtypes a family module names: each stored struct's +/// ([`stored_term_newtype`]) and each query struct's +/// ([`Term::binding_newtype`]), after `Ciphertext`, in first-use order. +fn family_terms<'d>( + eql: EqlVersion, + domains: impl Iterator, +) -> Vec<&'static str> { + let mut used: Vec<&'static str> = vec!["Ciphertext"]; + for d in domains { + for term in Term::payload_terms(d.terms) { + for t in [stored_term_newtype(eql, d, term), term.binding_newtype()] { + if !used.contains(&t) { + used.push(t); + } + } + } + } + used +} + +/// The name of the projection of `domain`'s query twin that carries `term` +/// alone: `TextSearchEqualityQuery`, `TextSearchOrderQuery`, +/// `TextSearchMatchQuery`. Named by what the term answers, not by its key, +/// so the order projection of the ORE twin would read `…OrderQuery` too. +pub fn query_projection_ident(family: &DomainFamily, domain: &Domain, term: Term) -> String { + let role = match term { + Term::Hm => "Equality", + Term::Ore | Term::Ope => "Order", + Term::Bloom => "Match", + }; + format!("{}{role}Query", domain.struct_ident(family.name)) +} + +/// For a producible EQL v4 query twin whose terms are optional, one +/// projection per term: a struct carrying the envelope and that term alone, +/// deriving `EncryptFrom` so it derives only that term, and converting into +/// the twin. Stack Encrypt builds the twin through the projection of the +/// index that answers the query's operator, so a query never derives a term +/// its operator does not read (a match term, which text shorter than one +/// 3-gram cannot make, is derived only for `@@`). Not domains: no +/// `DomainType`, not in the inventory. Empty for every other domain. +fn render_query_projections( + eql: EqlVersion, + family: &DomainFamily, + domain: &Domain, +) -> TokenStream { + if eql != EqlVersion::V4 + || !v4_optional_query_terms(domain) + || encryption_gap(eql, family, domain).is_some() + { + return TokenStream::new(); + } + let twin = format_ident!("{}Query", domain.struct_ident(family.name)); + let query_domain = format!("{}.{}", eql.prefix(), domain.query_name(family.name)); + let (_, rust) = plaintext_of(family.name).expect("a derived family names its plaintext"); + let plaintext_ty: syn::Type = syn::parse_str(rust).expect("plaintext is a type path"); + let shown = rust.rsplit("::").next().unwrap_or(rust); + let terms = Term::payload_terms(domain.terms); + terms + .iter() + .map(|&term| { + let ident = format_ident!("{}", query_projection_ident(family, domain, term)); + let fid = format_ident!("{}", term.json_key()); + let tid = format_ident!("{}", term.binding_newtype()); + let ops = Term::operators_for_terms(domain.terms) + .into_iter() + .filter(|op| Term::extractor_for_operator(domain.terms, op) == Some(term.extractor())) + .map(|op| format!("`{op}`")) + .collect::>() + .join(" "); + let summary = format!( + " The `{query_domain}` query operand for {ops}: the `{}` term alone.", + term.json_key() + ); + let detail = format!( + " Implements `EncryptFrom<{shown}>`, deriving only this term, and converts into \ + [`{twin}`], which is what is bound in SQL." + ); + let others: TokenStream = terms + .iter() + .filter(|&&other| other != term) + .map(|other| { + let oid = format_ident!("{}", other.json_key()); + quote!(#oid: None,) + }) + .collect(); + quote! { + #[doc = #summary] + #[doc = ""] + #[doc = #detail] + #[doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`."] + #[derive(stack_encrypt::EncryptFrom)] + #[stash(plaintext = #plaintext_ty)] + #[derive(Clone, Debug, PartialEq)] + pub struct #ident { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub #fid: #tid, + } + + impl From<#ident> for #twin { + fn from(query: #ident) -> Self { + #twin { + v: query.v, + i: query.i, + #fid: Some(query.#fid), + #others + } + } + } + } + }) + .collect() +} + /// The `(family, domain)` pairs whose generated v4 structs carry the opt-in /// `stack-encrypt` derives: the storage and equality domains of every family -/// whose plaintext eql-bindings names (`plaintext_of`), the OPE-ordered -/// domains (`ord`, `ord_ope`) of those whose order terms the engine derives, -/// and text match. Every other stored domain has a reason in -/// [`encryption_gap`], and a test holds the two together so a new domain +/// whose plaintext eql-bindings names (`plaintext_of`), the ordered domains +/// (`ord` and `ord_ope` with an OPE term, `ord_ore` with a block ORE term) of +/// those whose order terms the engine derives, and text match and search +/// (`search` and `search_ore`). Every other stored domain has a reason +/// in [`encryption_gap`], and a test holds the two together so a new domain /// cannot arrive without either a derive or a reason. +/// +/// Listed in catalog order, which is the order the generated target table +/// lists them in. pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ ("integer", ""), ("integer", "eq"), + ("integer", "ord_ore"), ("integer", "ord"), ("integer", "ord_ope"), ("smallint", ""), ("smallint", "eq"), + ("smallint", "ord_ore"), ("smallint", "ord"), ("smallint", "ord_ope"), ("bigint", ""), ("bigint", "eq"), + ("bigint", "ord_ore"), ("bigint", "ord"), ("bigint", "ord_ope"), ("date", ""), @@ -440,15 +608,20 @@ pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ ("text", ""), ("text", "eq"), ("text", "match"), + ("text", "ord_ore"), ("text", "ord"), ("text", "ord_ope"), + ("text", "search_ore"), + ("text", "search"), ("boolean", ""), ("real", ""), ("real", "eq"), + ("real", "ord_ore"), ("real", "ord"), ("real", "ord_ope"), ("double", ""), ("double", "eq"), + ("double", "ord_ore"), ("double", "ord"), ("double", "ord_ope"), ]; @@ -511,22 +684,13 @@ pub fn encryption_gap( } else if ORDER_PENDING_FAMILIES.contains(&family.name) && (domain.terms.contains(&Term::Ope) || domain.terms.contains(&Term::Ore)) { - // Before the ORE branch: the engine derives no order term of either - // scheme for these kinds, so block ORE is not the first thing a - // `*OrdOre` type of these families waits for. + // The engine derives no order term of either scheme for these kinds. "the engine derives no order term for a date, timestamp or decimal yet: how they are \ ordered is not decided (#1236)" - } else if domain.terms.contains(&Term::Ore) { - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)" - } else if domain.terms.contains(&Term::Bloom) && domain.terms.contains(&Term::Ope) { - "every query of this type carries a match term, which text shorter than one 3-gram \ - cannot make, so `=` or `<` on short text would be refused; how a query that is not a \ - match leaves it out is not decided (#1131)" } else { // Reached by any unlisted domain of a family whose plaintext is - // specified and whose terms are neither ORE nor match-and-order (an - // equality or OPE-ordered domain of a family that gains a plaintext, - // say): nothing in the catalog today. + // specified (a domain added to such a family, say): nothing in the + // catalog today. "this domain derives the way its family's other domains do; it is not listed in \ ENCRYPTION_DOMAINS yet" }) @@ -555,7 +719,7 @@ fn encryption_attrs( let terms = match ( domain.terms.contains(&Term::Bloom), domain.terms.contains(&Term::Hm), - domain.terms.contains(&Term::Ope), + domain.terms.contains(&Term::Ope) || domain.terms.contains(&Term::Ore), ) { (true, _, _) => "match", (false, true, true) => "equality and order", @@ -574,6 +738,28 @@ fn encryption_attrs( } }; let plaintext = plaintext_doc(&format!("[`{shown}`]({rust})")); + // The twin of an optional-term query is built by its projections + // (`render_query_projections`), one per index, so it carries no derive. + if query && v4_optional_query_terms(domain) { + let projections = Term::payload_terms(domain.terms) + .into_iter() + .map(|term| format!("[`{}`]", query_projection_ident(family, domain, term))) + .collect::>() + .join(", "); + let built = format!( + " Plaintext input: [`{shown}`]({rust}), through one projection per index — \ + {projections} — each `EncryptFrom<{shown}>` and `Into`: a query carries the \ + term of the index that answers its operator. Query operands do not decrypt." + ); + return ( + quote! { + #[doc = ""] + #[doc = #built] + }, + quote!(), + quote!(), + ); + } ( quote! { #[doc = ""] @@ -636,15 +822,7 @@ fn family_imports(version: EqlVersion, used: &[&'static str]) -> TokenStream { /// (exactly the term newtypes the family uses) followed by every domain's /// storage struct + impl, then a query twin for each term-bearing domain. pub fn render_family_bindings(version: EqlVersion, family: &DomainFamily) -> String { - let mut used: Vec<&'static str> = vec!["Ciphertext"]; - for d in family.domains { - for term in Term::payload_terms(d.terms) { - let t = term.binding_newtype(); - if !used.contains(&t) { - used.push(t); - } - } - } + let used = family_terms(version, family.domains.iter()); let imports = family_imports(version, &used); // Storage structs for every domain, then a query twin for each term-bearing @@ -698,15 +876,7 @@ pub fn render_family_bindings(version: EqlVersion, family: &DomainFamily) -> Str /// `json.rs` re-exports the generated struct (`pub use ...::Json;`) so /// `super::json::Json` resolves for the inventory/payload renderers. pub fn render_scalar_only_bindings(version: EqlVersion, family: &DomainFamily) -> String { - let mut used: Vec<&'static str> = vec!["Ciphertext"]; - for d in family.domains.iter().filter(|d| d.is_scalar()) { - for term in Term::payload_terms(d.terms) { - let t = term.binding_newtype(); - if !used.contains(&t) { - used.push(t); - } - } - } + let used = family_terms(version, family.domains.iter().filter(|d| d.is_scalar())); let imports = family_imports(version, &used); // Scalar storage structs only, then a query twin for each term-bearing scalar @@ -1204,7 +1374,8 @@ fn render_bindings(dir: &Path) -> Vec<(PathBuf, String)> { /// mixed family's scalar domains under its own name, since v4 has no /// hand-written SteVec module to sit beside), and the Stack Encrypt target /// table and dispatch (`targets.rs`, declared under the `targets` feature in -/// the hand-written `v4/mod.rs`). Stack Encrypt produces v4 only, so the +/// the hand-written `v4/mod.rs`), and the typed query chain's impls +/// (`answers.rs`, `stack-encrypt` alone). Stack Encrypt produces v4 only, so the /// target table lives here. fn render_v4_bindings(dir: &Path) -> Vec<(PathBuf, String)> { let mut rendered: Vec<(PathBuf, String)> = eql_domains::families_with_scalar_domains() @@ -1218,6 +1389,7 @@ fn render_v4_bindings(dir: &Path) -> Vec<(PathBuf, String)> { }) .collect(); rendered.push((dir.join("targets.rs"), crate::targets::render_targets_rs())); + rendered.push((dir.join("answers.rs"), crate::targets::render_answers_rs())); rendered.push((dir.join("inventory.rs"), render_v4_inventory_rs())); rendered } @@ -1458,6 +1630,50 @@ mod tests { ); } + /// A v4 search domain's stored filter is `SearchBloomFilter`, which takes + /// the empty filter of text too short for a 3-gram; `TextMatch` and every + /// query keep `BloomFilter`, which refuses that text. v3 is untouched. + #[test] + fn a_v4_search_domain_stores_a_filter_that_may_be_empty() { + let v4 = render_family_bindings(EqlVersion::V4, family("text")); + let field = |out: &str, name: &str| { + let start = out + .find(&format!("pub struct {name} {{")) + .unwrap_or_else(|| panic!("no struct {name}")); + let body = &out[start..]; + let body = &body[..body.find('}').unwrap()]; + body.lines() + .find(|line| line.contains("pub bf:")) + .map(|line| line.trim().to_owned()) + }; + assert_eq!( + field(&v4, "TextSearch").as_deref(), + Some("pub bf: SearchBloomFilter,") + ); + assert_eq!( + field(&v4, "TextSearchOre").as_deref(), + Some("pub bf: SearchBloomFilter,") + ); + assert_eq!( + field(&v4, "TextMatch").as_deref(), + Some("pub bf: BloomFilter,") + ); + assert_eq!( + field(&v4, "TextMatchQuery").as_deref(), + Some("pub bf: BloomFilter,") + ); + assert_eq!( + field(&v4, "TextSearchMatchQuery").as_deref(), + Some("pub bf: BloomFilter,") + ); + assert_eq!( + field(&v4, "TextSearchQuery").as_deref(), + Some("pub bf: Option,") + ); + let v3 = render_family_bindings(EqlVersion::V3, family("text")); + assert!(!v3.contains("SearchBloomFilter")); + } + #[test] fn bool_storage_only_family_has_one_struct_no_terms() { let out = render_family_bindings(EqlVersion::V3, family("boolean")); @@ -1507,7 +1723,7 @@ mod tests { // inventory. let families = eql_domains::scalar_families().count(); let v4_families = eql_domains::families_with_scalar_domains().count(); - assert_eq!(written.len(), (families + 4) + (v4_families + 2)); + assert_eq!(written.len(), (families + 4) + (v4_families + 3)); assert!(dir.join("integer.rs").is_file()); assert!(dir.join("text.rs").is_file()); assert!(dir.join("json_storage.rs").is_file()); @@ -1520,6 +1736,7 @@ mod tests { assert!(v4.join("text.rs").is_file()); assert!(v4.join("json.rs").is_file()); assert!(v4.join("targets.rs").is_file()); + assert!(v4.join("answers.rs").is_file()); assert!(v4.join("inventory.rs").is_file()); for d in [&dir, &v4] { assert!( @@ -1873,17 +2090,8 @@ mod tests { "no Rust plaintext type for this family" } else if ORDER_PENDING_FAMILIES.contains(&family.name) { "no order term for a date, timestamp or decimal" - } else if domain.terms.contains(&Term::Ore) { - "block ORE terms" - } else if domain.terms.contains(&Term::Bloom) - && domain.terms.contains(&Term::Ope) - { - "carries a match term" } else { - panic!( - "{full}: every specified domain without ORE or a match-and-order \ - query is derived: {reason}" - ) + panic!("{full}: every domain of a specified family is derived: {reason}") }; assert!(reason.contains(expected), "{full}: {reason}"); } @@ -1904,9 +2112,12 @@ mod tests { let attrs = encryption_attrs(EqlVersion::V4, family, domain, query) .0 .to_string(); + // A query whose terms are optional is built by its per-index + // projections, which derive in its place. + let projected = query && v4_optional_query_terms(domain); assert_eq!( attrs.contains("stack_encrypt :: EncryptFrom"), - listed, + listed && !projected, "{} (query = {query})", domain.full_name(family.name) ); diff --git a/packages/eql/crates/eql-codegen/src/consts.rs b/packages/eql/crates/eql-codegen/src/consts.rs index dbbae25a1..6cca18dff 100644 --- a/packages/eql/crates/eql-codegen/src/consts.rs +++ b/packages/eql/crates/eql-codegen/src/consts.rs @@ -48,6 +48,23 @@ pub(crate) const PUBLIC_TYPNAME_PREFIX: &str = "{{prefix}}_"; /// The payload version every domain CHECK compares `VALUE->>'v'` against. pub(crate) const EQL_VERSION: &str = "{{eql_version}}"; +/// The block ORE term type's name, which differs between the versions: +/// `ore_block_256` in `eql_v3`, `ore_block_v2` in `eql_v4` +/// ([`crate::render`]). [`eql_domains::Term::ctor`] stays the `eql_v3` name, +/// which the Rust bindings and the catalog dump report; the generated SQL +/// writes this placeholder in its place ([`sql_ctor`]). +pub(crate) const ORE_BLOCK: &str = "{{ore_block}}"; + +/// The constructor name a term's generated SQL calls: [`ORE_BLOCK`] for the +/// block ORE term, whose name follows the version, and +/// [`eql_domains::Term::ctor`] for the rest. +pub(crate) fn sql_ctor(term: eql_domains::Term) -> &'static str { + match term { + eql_domains::Term::Ore => ORE_BLOCK, + other => other.ctor(), + } +} + /// Always-present payload keys checked for presence in every domain CHECK. /// Term-specific keys are appended after these by `context::domain_block`. /// Defined in the catalog (`eql_domains::ENVELOPE_KEYS`) so the CHECKs and diff --git a/packages/eql/crates/eql-codegen/src/context.rs b/packages/eql/crates/eql-codegen/src/context.rs index a34cbd69b..67c820214 100644 --- a/packages/eql/crates/eql-codegen/src/context.rs +++ b/packages/eql/crates/eql-codegen/src/context.rs @@ -2,6 +2,7 @@ use crate::consts::*; use crate::operator_surface::Operator; +use crate::render::{block_close, block_open, EqlVersion}; use eql_domains::{Domain, Term}; /// Build the minijinja environment with the embedded templates: one whole-file @@ -72,9 +73,71 @@ pub fn environment() -> minijinja::Environment<'static> { env.add_global("schema", SCHEMA); env.add_global("internal_schema", INTERNAL_SCHEMA); env.add_global("eql_version", EQL_VERSION); + env.add_global("ore_block", ORE_BLOCK); + // The renderer's version-block directives, for SQL that differs between + // `eql_v3` and `eql_v4` (`crate::render`, "Version blocks"). + env.add_global("v3_only", block_open(EqlVersion::V3)); + env.add_global("v3_end", block_close(EqlVersion::V3)); + env.add_global("v4_only", block_open(EqlVersion::V4)); + env.add_global("v4_end", block_close(EqlVersion::V4)); env } +/// Whether an EQL v4 query of `domain` carries only the terms its operator +/// reads (ADR-0002, amended 2026-10-19): a domain whose terms include the +/// match term beside others, `text_search` and `text_search_ore`. Text +/// shorter than one 3-gram has no match term, so a query that had to carry +/// every term could not express `=` or `<` on it. +/// +/// EQL v4 only. The v4 query domain then requires at least one of its terms, +/// and each extractor over it raises when the term it reads is absent +/// ([`required_term`]); the `eql_v3` domain is cipherstash-client's format and +/// keeps requiring every term. The v4 binding's query struct holds each term +/// as an `Option`. +pub fn v4_optional_query_terms(domain: &Domain) -> bool { + domain.terms.contains(&Term::Bloom) && Term::payload_terms(domain.terms).len() > 1 +} + +/// The term an extractor over a query operand reads, for an operand whose +/// terms are optional in EQL v4 ([`v4_optional_query_terms`]): the extractor +/// raises, naming the term and the operators that read it, when the operand +/// carries none, rather than answer `NULL` and match no row. +#[derive(serde::Serialize)] +pub struct RequiredTerm { + /// sql_str-escaped JSON key, e.g. `hm`. + pub key: String, + /// sql_str-escaped schema-qualified query domain, for the message. + pub domain: String, + /// sql_str-escaped operators the term answers on this domain, as prose: + /// `= and <>`, `<, <=, > and >=`, `@@`. + pub operators: String, +} + +/// The [`RequiredTerm`] for `term`'s extractor over `domain`'s query operand +/// `query_dom`, or `None` when the operand's terms are not optional. +pub fn required_term(domain: &Domain, term: Term, query_dom: &str) -> Option { + if !v4_optional_query_terms(domain) { + return None; + } + // The operators this extractor answers on this domain: an operator two + // terms support is read through the first (`=` on `text_search` reads + // `hm`, not `op`). + let ops: Vec<&str> = Term::operators_for_terms(domain.terms) + .into_iter() + .filter(|op| Term::extractor_for_operator(domain.terms, op) == Some(term.extractor())) + .collect(); + let operators = match ops.as_slice() { + [] => "no operator".to_string(), + [one] => (*one).to_string(), + [rest @ .., last] => format!("{} and {last}", rest.join(", ")), + }; + Some(RequiredTerm { + key: sql_str(term.json_key()), + domain: sql_str(query_dom), + operators: sql_str(&operators), + }) +} + /// One idempotent CREATE DOMAIN block, with SQL-required values precomputed. #[derive(serde::Serialize)] pub struct DomainBlock { @@ -91,6 +154,13 @@ pub struct DomainBlock { // query operand is index-terms-only). The template renders a // `NOT (VALUE ? k)` clause per key. pub forbidden_keys: Vec, + // sql_str-escaped term keys of a query twin whose terms are optional in + // EQL v4 (`v4_optional_query_terms`); empty otherwise. When set, `keys` + // holds the envelope keys alone, and the template renders the term keys + // as required for `eql_v3` and as "at least one" for `eql_v4`, each in + // its own version block, with a present `nonempty_array_keys` key still + // a non-empty array. + pub optional_term_keys: Vec, // sql_str-escaped one-line human description rendered as `COMMENT ON DOMAIN`. // Capability text is derived from the domain's terms (via // `Term::operators_for_terms`), so it can't drift from the generated CHECK / @@ -171,6 +241,8 @@ pub fn domain_block(family_name: &str, domain: &Domain) -> DomainBlock { .collect(), // Storage domains forbid nothing; the query twin forbids `c`. forbidden_keys: vec![], + // A stored value carries every term in both versions. + optional_term_keys: vec![], comment: scalar_domain_comment(family_name, domain), } } @@ -188,9 +260,13 @@ pub fn query_domain_block(family_name: &str, domain: &Domain) -> DomainBlock { .filter(|&&k| k != "c") .map(|k| sql_str(k)) .collect(); - for k in Term::term_json_keys(domain.terms) { - keys.push(sql_str(k)); - } + let term_keys = Term::term_json_keys(domain.terms).into_iter().map(sql_str); + let optional_term_keys = if v4_optional_query_terms(domain) { + term_keys.collect() + } else { + keys.extend(term_keys); + vec![] + }; DomainBlock { typname: sql_str(&name), @@ -201,6 +277,7 @@ pub fn query_domain_block(family_name: &str, domain: &Domain) -> DomainBlock { .map(sql_str) .collect(), forbidden_keys: vec![sql_str("c")], + optional_term_keys, comment: query_domain_comment(family_name, domain), } } @@ -223,6 +300,9 @@ pub enum FnEntry { ret: String, // e.g. eql_v3_internal.hmac_256 (selection STAYS in Rust) extractor: String, // e.g. eq_term ctor: String, // e.g. hmac_256 (called as {{ internal_schema }}.{{ ctor }}) + // Set on an extractor over a query operand whose terms are optional + // in EQL v4: the v4 body raises when the term is absent. + required: Option, }, Wrapper { op: String, // SQL operator used in the body, e.g. = @@ -256,14 +336,26 @@ pub struct FunctionsContext { /// Build the inlinable index-extractor entry for a domain term. /// /// The `RETURNS` type name equals the constructor name (`hmac_256`, -/// `ore_block_256`); qualify it with `INTERNAL_SCHEMA` — the same schema as +/// `{{ore_block}}`, [`sql_ctor`]); qualify it with `INTERNAL_SCHEMA` — the same schema as /// the body's constructor call — so the declared return type and the call /// stay in lockstep. `Term::returns()` is intentionally not used. pub fn extractor_entry(term: Term) -> FnEntry { + extractor_entry_requiring(term, None) +} + +/// [`extractor_entry`] over a query operand: in EQL v4 it raises when the +/// operand lacks the term, if the operand's terms are optional there +/// ([`required_term`]). +pub fn query_extractor_entry(domain: &Domain, term: Term, query_dom: &str) -> FnEntry { + extractor_entry_requiring(term, required_term(domain, term, query_dom)) +} + +fn extractor_entry_requiring(term: Term, required: Option) -> FnEntry { FnEntry::Extractor { - ret: format!("{INTERNAL_SCHEMA}.{}", term.ctor()), + ret: format!("{INTERNAL_SCHEMA}.{}", sql_ctor(term)), extractor: term.extractor().to_string(), - ctor: term.ctor().to_string(), + ctor: sql_ctor(term).to_string(), + required, } } diff --git a/packages/eql/crates/eql-codegen/src/generate.rs b/packages/eql/crates/eql-codegen/src/generate.rs index b6df602af..ce35b1cb7 100644 --- a/packages/eql/crates/eql-codegen/src/generate.rs +++ b/packages/eql/crates/eql-codegen/src/generate.rs @@ -258,8 +258,8 @@ fn query_types_path(family_name: &str) -> String { pub fn render_query_functions_file(family_name: &str, domain: &Domain) -> String { use crate::consts::sql_str; use crate::context::{ - domain_name, environment, extractor_entry, query_domain_name, wrapper_entry, - FunctionsContext, + domain_name, environment, query_domain_name, query_extractor_entry, + v4_optional_query_terms, wrapper_entry, FunctionsContext, }; let name = domain.full_name(family_name); let query_name = domain.query_name(family_name); @@ -269,9 +269,10 @@ pub fn render_query_functions_file(family_name: &str, domain: &Domain) -> String let mut entries = Vec::new(); // Extractor overloads on the query domain (the template renders `a {{ dom }}` - // with dom = the query domain). + // with dom = the query domain). Over an operand whose terms are optional + // in EQL v4, each raises there when its term is absent. for term in Term::extractor_terms(domain.terms) { - entries.push(extractor_entry(term)); + entries.push(query_extractor_entry(domain, term, &query_dom)); } // Comparison wrappers: (storage, query) and its (query, storage) commutator, // for supported operators only (a query operand is never sent for a blocked @@ -307,12 +308,18 @@ pub fn render_query_functions_file(family_name: &str, domain: &Domain) -> String )); } + let mut requires = vec![ + V3_SCHEMA.to_string(), + query_types_path(family_name), + scalar_path(family_name, &format!("{name}_functions.sql")), + ]; + // The v4 extractors over an operand with optional terms raise through the + // shared helper there. + if v4_optional_query_terms(domain) { + requires.push(V3_SCALARS_BLOCKER.to_string()); + } let ctx = FunctionsContext { - requires: vec![ - V3_SCHEMA.to_string(), - query_types_path(family_name), - scalar_path(family_name, &format!("{name}_functions.sql")), - ], + requires, family_name: family_name.to_string(), name: query_name, domain_lit: sql_str(&query_dom), @@ -2529,4 +2536,106 @@ mod tests { ); } } + + /// The CHECK of `name`'s CREATE DOMAIN in `sql`, rendered as `prefix`. + fn domain_check(sql: &str, prefix: &str, name: &str) -> String { + let start = sql + .find(&format!("CREATE DOMAIN {prefix}.{name} AS")) + .unwrap_or_else(|| panic!("{prefix}.{name}")); + let end = start + sql[start..].find(");").expect("CHECK closes"); + sql[start..end].to_string() + } + + /// EQL v4's search query operands carry only the terms their operator + /// reads (ADR-0002, amended 2026-10-19); eql_v3's keep requiring every + /// term, as before. + #[test] + fn a_v4_search_query_operand_needs_one_term_and_v3_needs_all() { + use crate::render::{render_str, EqlVersion}; + let text = spec("text"); + let types = super::render_query_types_file(text); + let v3 = render_str(&types, EqlVersion::V3).unwrap(); + let v4 = render_str(&types, EqlVersion::V4).unwrap(); + + let search3 = domain_check(&v3, "eql_v3", "query_text_search"); + for key in ["hm", "op", "bf"] { + assert!( + search3.contains(&format!("AND VALUE ? '{key}'")), + "{search3}" + ); + } + let search4 = domain_check(&v4, "eql_v4", "query_text_search"); + assert!( + search4.contains("AND (VALUE ? 'hm' OR VALUE ? 'op' OR VALUE ? 'bf')"), + "{search4}" + ); + assert!(!search4.contains("AND VALUE ? 'hm'"), "{search4}"); + assert!( + search4.contains("AND VALUE ? 'v'") && search4.contains("NOT (VALUE ? 'c')"), + "{search4}" + ); + // The ORE twin: a present `ob` is still a non-empty array. + let ore3 = domain_check(&v3, "eql_v3", "query_text_search_ore"); + assert!(ore3.contains("AND VALUE ? 'ob'"), "{ore3}"); + assert!( + ore3.contains("AND jsonb_typeof(VALUE -> 'ob') = 'array'"), + "{ore3}" + ); + let ore4 = domain_check(&v4, "eql_v4", "query_text_search_ore"); + assert!( + ore4.contains("AND (VALUE ? 'hm' OR VALUE ? 'ob' OR VALUE ? 'bf')") + && ore4 + .contains("AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'array'"), + "{ore4}" + ); + // Every other query operand is the same in both versions. + for other in ["query_text_eq", "query_text_match", "query_text_ord"] { + assert_eq!( + domain_check(&v3, "eql_v3", other) + .replace("eql_v3", "eql_vN") + .replace("'3'", "'N'"), + domain_check(&v4, "eql_v4", other) + .replace("eql_v4", "eql_vN") + .replace("'4'", "'N'"), + "{other}" + ); + } + + // Each v4 extractor over the operand raises without its term; v3's + // reads the term as before. + let functions = super::render_query_functions_file("text", domain(text, "search")); + let f3 = render_str(&functions, EqlVersion::V3).unwrap(); + let f4 = render_str(&functions, EqlVersion::V4).unwrap(); + assert!(!f3.contains("require_query_term"), "{f3}"); + assert!(f3.contains("AS $$ SELECT eql_v3_internal.hmac_256(a::jsonb) $$;")); + for (ctor, key, ops) in [ + ("hmac_256", "hm", "= and <>"), + ("ope_cllw", "op", "<, <=, > and >="), + ("bloom_filter", "bf", "@@"), + ] { + let body = format!( + "SELECT eql_v4_internal.{ctor}(eql_v4_internal.require_query_term(a::jsonb, \ + 'eql_v4.query_text_search', '{key}', '{ops}'))" + ); + assert!(f4.contains(&body), "{body} in {f4}"); + } + // A query operand whose terms are not optional reads them directly, + // in one form for both versions. + let eq = super::render_query_functions_file("text", domain(text, "eq")); + assert!(!eq.contains("require_query_term") && !eq.contains("{{#v")); + } + + #[test] + fn only_the_search_domains_have_optional_v4_query_terms() { + let optional: Vec = CATALOG + .iter() + .flat_map(|f| { + f.domains + .iter() + .filter(|d| d.is_scalar() && crate::context::v4_optional_query_terms(d)) + .map(move |d| d.full_name(f.name)) + }) + .collect(); + assert_eq!(optional, ["text_search_ore", "text_search"]); + } } diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 1f4d41634..4ef57020e 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -241,10 +241,13 @@ mod tests { // so no type is declared twice. assert_eq!(out.matches("\ntype JSON []byte\n").count(), 1); assert_eq!(out.matches("\ntype SteVecQuery []byte\n").count(), 1); - // TextEq is producible and says so; TextOrdOre is not and says why. + // TextEq is producible and says so; DateOrdOre is not and says why. assert!(out.contains("// TextEq is the EQL type stored in public.eql_v4_text_eq")); assert!(out.contains("Producible: true, Reason: \"\"")); - assert!(out.contains("Name: \"TextOrdOre\"") && out.contains("block ORE terms")); + assert!( + out.contains("Name: \"DateOrdOre\"") + && out.contains("no order term for a date, timestamp or decimal") + ); // Every table row is one line: the shape gofmt leaves alone. for line in out.lines().filter(|l| l.starts_with("\t{Name: ")) { assert!(line.ends_with("},"), "{line}"); @@ -269,7 +272,7 @@ mod tests { } assert_eq!(count, 18, "nine ordered families, ORE and OPE each"); assert!(out.contains( - "// IntegerRangeOreQuery is a range query value over the ore index of the integer family: \ + "// IntegerRangeOreQuery is a range query value over the ore_block index of the integer family: \ both bounds of one BETWEEN,\n// included, as the one operand eql_v4.query_integer_range_ore \ takes in eql_v4.between. It ranges over IntegerOrdOre.\n" )); diff --git a/packages/eql/crates/eql-codegen/src/range.rs b/packages/eql/crates/eql-codegen/src/range.rs index 8c31f5d13..a231c7c9e 100644 --- a/packages/eql/crates/eql-codegen/src/range.rs +++ b/packages/eql/crates/eql-codegen/src/range.rs @@ -267,7 +267,7 @@ pub fn render_range_functions_file(spec: &DomainFamily) -> Option { }; for &term in &terms { let range = spec.range_query_name(term); - let returns = format!("{INTERNAL_SCHEMA}.{}", term.ctor()); + let returns = format!("{INTERNAL_SCHEMA}.{}", crate::consts::sql_ctor(term)); for (side, function_name, bound) in [ ("low", "lo", RANGE_BOUND_KEYS[0]), ("high", "hi", RANGE_BOUND_KEYS[1]), @@ -280,7 +280,7 @@ pub fn render_range_functions_file(spec: &DomainFamily) -> Option { returns: returns.clone(), body: format!( "{INTERNAL_SCHEMA}.{}(jsonb_build_object('{}', r::jsonb -> '{}'))", - term.ctor(), + crate::consts::sql_ctor(term), sql_str(term.json_key()), sql_str(bound) ), @@ -456,9 +456,9 @@ mod tests { for (side, bound) in [("lo", "lo"), ("hi", "hi")] { assert!(v4.contains(&format!( "CREATE FUNCTION eql_v4_internal.{side}(r eql_v4.query_integer_range_ore)\n\ - RETURNS eql_v4_internal.ore_block_256\n\ + RETURNS eql_v4_internal.ore_block_v2\n\ LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n\ - AS $$ SELECT eql_v4_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> '{bound}')) $$;" + AS $$ SELECT eql_v4_internal.ore_block_v2(jsonb_build_object('ob', r::jsonb -> '{bound}')) $$;" )), "{side} ORE"); assert!(v4.contains(&format!( "CREATE FUNCTION eql_v4_internal.{side}(r eql_v4.query_integer_range_ope)\n\ @@ -582,8 +582,9 @@ mod tests { )), "{v4}" ); - assert!(v4 - .contains("AND c.opcintype = 'eql_v4_internal.ore_block_256'::pg_catalog.regtype")); + assert!( + v4.contains("AND c.opcintype = 'eql_v4_internal.ore_block_v2'::pg_catalog.regtype") + ); } } diff --git a/packages/eql/crates/eql-codegen/src/render.rs b/packages/eql/crates/eql-codegen/src/render.rs index 6f39f3b01..c37247164 100644 --- a/packages/eql/crates/eql-codegen/src/render.rs +++ b/packages/eql/crates/eql-codegen/src/render.rs @@ -10,10 +10,19 @@ //! |-------------------|----------|----------| //! | `{{prefix}}` | `eql_v3` | `eql_v4` | //! | `{{eql_version}}` | `3` | `4` | +//! | `{{ore_block}}` | `ore_block_256` | `ore_block_v2` | //! //! `{{prefix}}` is both the schema (`{{prefix}}.eq_term`) and the start of every //! other name (`{{prefix}}_internal`, `public.{{prefix}}_text_eq`). //! +//! `{{ore_block}}` names the block ORE term type and everything built on it +//! (`{{prefix}}_internal.{{ore_block}}_term`, `compare_{{ore_block}}_terms`). +//! The two versions compare different ciphertexts under the one `ob` key: +//! `eql_v3` cipherstash-client's 8-bit block ORE, whose blocks are 256-way, +//! and `eql_v4` ore-rs's 6-bit block ORE, whose wire format is ore-rs's +//! version 2. Each version's name says which it compares, so `eql_v4` does +//! not reuse the name of another format. +//! //! Rendering is strict in both directions. An unknown or unterminated //! placeholder is an error, and so is a literal `eql_v3` or `eql_v4` in the //! source: a literal name would render the same in both outputs, so the `eql_v4` @@ -25,6 +34,26 @@ //! `between`, ADR-0009 decision 7) shares the one source tree: the build //! renders a file only for the version it belongs to ([`belongs_to`]), and //! [`render_str`] refuses to render it as any other. +//! # Version blocks +//! +//! Where one version's SQL must differ from the other's, the source holds both +//! forms, each between two directive lines (ADR-0002, amended 2026-10-19): +//! +//! ```sql +//! -- {{#v3}} +//! AND VALUE ? 'hm' +//! -- {{/v3}} +//! -- {{#v4}} +//! AND (VALUE ? 'hm' OR VALUE ? 'op') +//! -- {{/v4}} +//! ``` +//! +//! A block's lines are kept in its own version's output and dropped from the +//! other's, and the directive lines are dropped from both, so a source with a +//! block renders, for the version the block does not name, exactly as it did +//! before the block was written. A directive is a whole line (leading +//! whitespace allowed); blocks do not nest, a close must name the open block's +//! version, and a block left open at the end of the file is an error. use std::fmt; use std::fs; @@ -42,6 +71,56 @@ pub const EQL_VERSION: &str = "{{eql_version}}"; /// to every version. The number is the payload version ([`EqlVersion::number`]), /// not the schema name, because the source may not name a schema literally. pub const ONLY_DIRECTIVE: &str = "-- ONLY EQL VERSION:"; +/// The block ORE name placeholder. See [the module docs](self). +pub const ORE_BLOCK: &str = "{{ore_block}}"; + +/// The `eql_v3` block ORE name, which only [`ORE_BLOCK`] may write: a literal +/// would name the 8-bit format in the `eql_v4` bundle too. The source +/// directory of that name (`src/v3/sem/ore_block_256/`) is a path, not a +/// name, and is allowed. +const V3_ORE_BLOCK: &str = "ore_block_256"; + +/// The directive line that opens a block kept only in `version`'s output: +/// `-- {{#v4}}`. See [the module docs](self#version-blocks). +pub fn block_open(version: EqlVersion) -> String { + format!("-- {{{{#v{}}}}}", version.number()) +} + +/// The directive line that closes a block [`block_open`] opened: `-- {{/v4}}`. +pub fn block_close(version: EqlVersion) -> String { + format!("-- {{{{/v{}}}}}", version.number()) +} + +/// A version-block directive line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Directive { + Open(EqlVersion), + Close(EqlVersion), +} + +/// The directive `line` is, if it is one: `-- {{#vN}}` or `-- {{/vN}}` for a +/// version this renderer knows, alone on the line. +fn directive(line: &str) -> Option { + let inner = line + .trim() + .strip_prefix("--")? + .trim_start() + .strip_prefix("{{")? + .strip_suffix("}}")?; + let (open, name) = if let Some(name) = inner.strip_prefix('#') { + (true, name) + } else { + (false, inner.strip_prefix('/')?) + }; + let version = EqlVersion::ALL + .into_iter() + .find(|v| name.strip_prefix('v') == Some(v.number()))?; + Some(if open { + Directive::Open(version) + } else { + Directive::Close(version) + }) +} /// An EQL version the source can be rendered as. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -71,6 +150,15 @@ impl EqlVersion { EqlVersion::V4 => "eql_v4", } } + + /// The block ORE name: `ore_block_256` (cipherstash-client's 8-bit + /// format) or `ore_block_v2` (ore-rs's 6-bit format, wire version 2). + pub fn ore_block(self) -> &'static str { + match self { + EqlVersion::V3 => V3_ORE_BLOCK, + EqlVersion::V4 => "ore_block_v2", + } + } } impl std::str::FromStr for EqlVersion { @@ -103,6 +191,15 @@ pub enum RenderError { only: EqlVersion, version: EqlVersion, }, + /// A literal `ore_block_256` where `{{ore_block}}` belongs. + LiteralOreBlock { line: usize }, + /// A version block opened inside another; blocks do not nest. + NestedBlock { line: usize }, + /// A block close with no open block, or naming another version than the + /// open one. + UnmatchedClose { line: usize }, + /// A block still open at the end of the source; `line` is its opening. + UnclosedBlock { line: usize }, } impl fmt::Display for RenderError { @@ -110,7 +207,8 @@ impl fmt::Display for RenderError { match self { RenderError::UnknownPlaceholder { line, name } => write!( f, - "line {line}: unknown placeholder {{{{{name}}}}} (expected {PREFIX} or {EQL_VERSION})" + "line {line}: unknown placeholder {{{{{name}}}}} (expected {PREFIX}, {EQL_VERSION} \ + or {ORE_BLOCK})" ), RenderError::Unterminated { line } => { write!(f, "line {line}: `{{{{` with no closing `}}}}`") @@ -139,6 +237,24 @@ impl fmt::Display for RenderError { only.prefix(), version.prefix() ), + RenderError::LiteralOreBlock { line } => write!( + f, + "line {line}: literal `{V3_ORE_BLOCK}`; write {ORE_BLOCK} in its place, in \ + comments too, so the name follows the EQL version's block ORE format" + ), + RenderError::NestedBlock { line } => { + write!(f, "line {line}: a version block opened inside another") + } + RenderError::UnmatchedClose { line } => write!( + f, + "line {line}: a version block close with no open block of that version" + ), + RenderError::UnclosedBlock { line } => { + write!( + f, + "line {line}: a version block opened here is never closed" + ) + } } } } @@ -234,6 +350,8 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result } } let mut out = String::with_capacity(src.len()); + // The open version block, and the line that opened it. + let mut block: Option<(EqlVersion, usize)> = None; for (i, line) in src.split_inclusive('\n').enumerate() { let lineno = i + 1; for v in EqlVersion::ALL { @@ -244,6 +362,27 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result }); } } + if contains_ore_block_name(line) { + return Err(RenderError::LiteralOreBlock { line: lineno }); + } + match (directive(line), block) { + (Some(Directive::Open(_)), Some(_)) => { + return Err(RenderError::NestedBlock { line: lineno }) + } + (Some(Directive::Open(v)), None) => { + block = Some((v, lineno)); + continue; + } + (Some(Directive::Close(v)), Some((open, _))) if v == open => { + block = None; + continue; + } + (Some(Directive::Close(_)), _) => { + return Err(RenderError::UnmatchedClose { line: lineno }) + } + (None, Some((v, _))) if v != version => continue, + (None, _) => {} + } let mut rest = line; while let Some(open) = rest.find("{{") { out.push_str(&rest[..open]); @@ -254,6 +393,7 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result match &after[..close] { "prefix" => out.push_str(version.prefix()), "eql_version" => out.push_str(version.number()), + "ore_block" => out.push_str(version.ore_block()), other => { return Err(RenderError::UnknownPlaceholder { line: lineno, @@ -265,6 +405,9 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result } out.push_str(rest); } + if let Some((_, line)) = block { + return Err(RenderError::UnclosedBlock { line }); + } Ok(out) } @@ -280,6 +423,14 @@ fn contains_name(line: &str, prefix: &str) -> bool { }) } +/// Whether `line` names `ore_block_256` other than as the source directory +/// `sem/ore_block_256/`. +fn contains_ore_block_name(line: &str) -> bool { + line.match_indices(V3_ORE_BLOCK).any(|(at, _)| { + !(line[..at].ends_with("sem/") && line[at + V3_ORE_BLOCK.len()..].starts_with('/')) + }) +} + /// Why a file could not be rendered. #[derive(Debug)] pub enum RenderFileError { @@ -428,6 +579,35 @@ mod tests { ); } + /// The block ORE name follows the version, and a literal of the + /// `eql_v3` name is refused unless it is the source directory's path. + #[test] + fn the_block_ore_name_follows_the_version() { + let src = "CREATE TYPE {{prefix}}_internal.{{ore_block}}_term AS (bytes bytea);\n\ + -- REQUIRE: src/v3/sem/ore_block_256/types.sql\n"; + assert_eq!( + render_str(src, EqlVersion::V3).unwrap(), + "CREATE TYPE eql_v3_internal.ore_block_256_term AS (bytes bytea);\n\ + -- REQUIRE: src/v3/sem/ore_block_256/types.sql\n" + ); + assert_eq!( + render_str(src, EqlVersion::V4).unwrap(), + "CREATE TYPE eql_v4_internal.ore_block_v2_term AS (bytes bytea);\n\ + -- REQUIRE: src/v3/sem/ore_block_256/types.sql\n" + ); + for literal in [ + "SELECT {{prefix}}_internal.ore_block_256(x);\n", + "-- compares ore_block_256 terms\n", + "-- the directory sem/ore_block_256 alone\n", + ] { + assert_eq!( + render_str(literal, EqlVersion::V4), + Err(RenderError::LiteralOreBlock { line: 1 }), + "{literal:?}" + ); + } + } + #[test] fn rejects_unterminated_placeholder() { assert_eq!( @@ -553,6 +733,94 @@ mod tests { ); } + #[test] + fn a_version_block_renders_in_its_own_version_only() { + let src = format!( + "a {{{{prefix}}}}\n{}\n three\n{}\n {}\n four {{{{eql_version}}}}\n {}\nz\n", + block_open(EqlVersion::V3), + block_close(EqlVersion::V3), + block_open(EqlVersion::V4), + block_close(EqlVersion::V4), + ); + assert_eq!( + render_str(&src, EqlVersion::V3).unwrap(), + "a eql_v3\n three\nz\n" + ); + assert_eq!( + render_str(&src, EqlVersion::V4).unwrap(), + "a eql_v4\n four 4\nz\n" + ); + } + + /// A block written for one version leaves the other's output exactly as + /// the source without the block renders it. + #[test] + fn a_block_for_one_version_leaves_the_other_unchanged() { + let plain = "SELECT {{prefix}}.f();\nSELECT 2;\n"; + let with = "SELECT {{prefix}}.f();\n-- {{#v4}}\nSELECT 'v4 only';\n-- {{/v4}}\nSELECT 2;\n"; + assert_eq!( + render_str(with, EqlVersion::V3).unwrap(), + render_str(plain, EqlVersion::V3).unwrap() + ); + assert_eq!( + render_str(with, EqlVersion::V4).unwrap(), + "SELECT eql_v4.f();\nSELECT 'v4 only';\nSELECT 2;\n" + ); + } + + #[test] + fn rejects_malformed_blocks() { + for (src, err) in [ + ( + "-- {{#v3}}\n-- {{#v4}}\n-- {{/v4}}\n-- {{/v3}}\n", + RenderError::NestedBlock { line: 2 }, + ), + ("x\n-- {{/v3}}\n", RenderError::UnmatchedClose { line: 2 }), + ( + "-- {{#v3}}\n-- {{/v4}}\n", + RenderError::UnmatchedClose { line: 2 }, + ), + ("x\n-- {{#v4}}\ny\n", RenderError::UnclosedBlock { line: 2 }), + ] { + assert_eq!(render_str(src, EqlVersion::V4), Err(err), "{src:?}"); + } + // A directive naming a version the renderer does not know, or one + // that is not alone on its line, is an unknown placeholder. + for src in ["-- {{#v5}}\n", "SELECT 1; -- {{#v4}}\n"] { + assert!( + matches!( + render_str(src, EqlVersion::V4), + Err(RenderError::UnknownPlaceholder { line: 1, .. }) + ), + "{src:?}" + ); + } + // A literal name inside the other version's block is still refused. + assert!(matches!( + render_str( + "-- {{#v4}}\nSELECT eql_v3.f();\n-- {{/v4}}\n", + EqlVersion::V3 + ), + Err(RenderError::LiteralName { line: 2, .. }) + )); + } + + #[test] + fn a_directive_may_be_indented() { + assert_eq!( + directive(" -- {{#v4}}\n"), + Some(Directive::Open(EqlVersion::V4)) + ); + assert_eq!( + directive("--{{/v3}}"), + Some(Directive::Close(EqlVersion::V3)) + ); + assert_eq!(block_open(EqlVersion::V4), "-- {{#v4}}"); + assert_eq!(block_close(EqlVersion::V3), "-- {{/v3}}"); + assert_eq!(directive("-- {{prefix}}"), None); + assert_eq!(directive("-- {{#v4}} trailing"), None); + } + #[test] fn parses_versions() { assert_eq!("3".parse(), Ok(EqlVersion::V3)); diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index c8fe66c67..6019c73b1 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -18,34 +18,54 @@ //! the derive, and [`target_gap`] is tested to agree with //! [`encryption_gap`](crate::bindings::encryption_gap) on every domain. //! +//! The same rows give the operators each producible type answers in Stack +//! Encrypt's typed query chain ([`answers_for_indexes`]), rendered to +//! `v4/answers.rs` ([`render_answers_rs`]) under the `stack-encrypt` feature +//! alone: it implements `op::Answers` for each, with the type's query twin as +//! the query term, and lists them in `ANSWERS` so the encryption tests can +//! hold them to the operators the engine's data path answers for the same +//! type. A producible type whose twin carries an order index, or more than +//! one index, answers nothing there yet ([`typed_gap`]): the chain sends one +//! index's term as the twin, and those twins need an order term, or several +//! terms at once. +//! //! The reasons a type is not producible: eql-bindings names no Rust //! plaintext type for the family; the engine derives no order term for a -//! date, timestamp or decimal; EQL's ORE term is block ORE where the engine -//! derives CLLW ORE (#1132); a -//! `TextSearch` query carries a match term that short text cannot make -//! (#1131); and the JSON index is a new engine operation (#1060). +//! date, timestamp or decimal; and the JSON index is a new engine +//! operation. Each reason string names the issue that tracks it. +//! +//! The query dispatch takes the index that answers the query's operator. A +//! twin whose terms are optional in EQL v4 (`TextSearchQuery`) is built +//! through the projection for that index, so it carries that term alone. use proc_macro2::TokenStream; use quote::{format_ident, quote}; use eql_domains::{Domain, DomainFamily, Shape, Term}; -use crate::bindings::{encryption_gap, format_rs, plaintext_of, stored_payload_domains}; +use crate::bindings::{ + encryption_gap, format_rs, plaintext_of, query_projection_ident, stored_payload_domains, +}; +use crate::context::v4_optional_query_terms; use crate::render::EqlVersion; /// The version every target is produced into. const TARGET_VERSION: EqlVersion = EqlVersion::V4; /// The `IndexSpec::key()` a catalog term rides under in a plan — the -/// `"eq"` / `"match"` / `"ore"` / `"ope"` strings of stack-encrypt's data -/// grammar — so a Go generator can match a target's indexes against the +/// `"eq"` / `"match"` / `"ore_block"` / `"ope"` strings of stack-encrypt's +/// data grammar — so a Go generator can match a target's indexes against the /// indexes a plan field may ask for by the same names. Exhaustive on /// purpose: a new catalog term must say which engine index it is. +/// +/// EQL's `ob` term is block ORE, the engine's `ore_block` index (ore-rs's +/// 6-bit schemes), not its `ore` index, which is CLLW ORE and has no EQL +/// domain. pub fn index_key(term: Term) -> &'static str { match term { Term::Hm => "eq", Term::Bloom => "match", - Term::Ore => "ore", + Term::Ore => "ore_block", Term::Ope => "ope", } } @@ -55,6 +75,83 @@ pub fn index_key(term: Term) -> &'static str { /// payload keys. const JSON_INDEX_KEY: &str = "json"; +/// An operator a producible EQL type answers in Stack Encrypt's typed query +/// chain: what the generated file implements `op::Answers` for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Answer { + /// `op::Eq`, through the equality index. + Eq, + /// `op::Matches`, through the match index under EQL's configuration. + Matches, +} + +impl Answer { + /// The operator's wire name, `stack_encrypt::dynamic::OpSpec::name`. + pub(crate) fn name(self) -> &'static str { + match self { + Answer::Eq => "eq", + Answer::Matches => "matches", + } + } + + /// The catalog term the operator's index derives: the one term the + /// query twin carries, under its JSON key. + pub(crate) fn term(self) -> Term { + match self { + Answer::Eq => Term::Hm, + Answer::Matches => Term::Bloom, + } + } +} + +/// Why the typed query chain has no answer yet for a type carrying these +/// indexes, by [`index_key`] name, or `None` when it answers: a query twin +/// is assembled from the one term its type's index derived, so a twin that +/// needs an order term (`"ore"`, `"ore_block"` or `"ope"`), or more than one term, is not +/// built by it. The reason is the generator's, for its tests and docs; the +/// generated file simply leaves the type out. +pub(crate) fn typed_gap(indexes: &[&str]) -> Option<&'static str> { + if indexes + .iter() + .any(|key| matches!(*key, "ore" | "ore_block" | "ope")) + { + Some("its twin carries an order term, which the typed query chain does not derive yet") + } else if indexes.len() > 1 { + Some("its twin carries more than one term, which the typed query chain does not assemble yet") + } else { + None + } +} + +/// The operators a producible type with a query twin answers, given the +/// indexes it carries by [`index_key`] name, the same input the engine's +/// table reads: `Eq` for the `"eq"` index and `Matches` for the `"match"` +/// index, as `OpSpec::answered_by` answers a field that names the type +/// (`TargetDescriptor::operators`). Empty for a type the chain has no +/// answer for yet ([`typed_gap`]). The encryption tests compare the two, +/// type by type, and hold the gap to exactly those types. +/// +/// # Panics +/// +/// On an index key the generator does not know: a new catalog term must say +/// what the typed query chain does with it. +pub(crate) fn answers_for_indexes(name: &str, indexes: &[&str]) -> Vec { + if typed_gap(indexes).is_some() { + return Vec::new(); + } + indexes + .iter() + .map(|key| match *key { + "eq" => Answer::Eq, + "match" => Answer::Matches, + other => panic!( + "{name}: the typed query chain has no rule for the {other} index; extend \ + eql-codegen's answers_for_indexes and typed_gap" + ), + }) + .collect() +} + /// The plaintext a family's domains are produced from: see /// [`bindings::plaintext_of`](crate::bindings::plaintext_of). pub fn plaintext(family: &DomainFamily) -> Option<(&'static str, &'static str)> { @@ -82,7 +179,15 @@ pub(crate) struct Row { /// The query twin's struct identifier and SQL domain, if the type /// answers a query. pub(crate) query: Option<(String, String)>, + /// For a query twin whose terms are optional in EQL v4, the projection + /// that builds it for each index, by the index's key: the query dispatch + /// runs the one for the index that answers the query's operator. + /// Empty for every other twin, whose plan derives all its terms. + pub(crate) projections: Vec<(&'static str, String)>, pub(crate) reason: Option<&'static str>, + /// The operators the type answers in the typed query chain: none + /// unless it is producible and has a query twin ([`answers_for_indexes`]). + pub(crate) answers: Vec, } /// Every row, in catalog order: one per stored domain. @@ -146,8 +251,15 @@ fn row(family: &'static DomainFamily, domain: &'static Domain) -> Row { ), )) }; + let name = domain.rust_struct_name(family.name); + let reason = target_gap(family, domain); + let answers = if reason.is_none() && query.is_some() { + answers_for_indexes(&name, &indexes) + } else { + Vec::new() + }; Row { - name: domain.rust_struct_name(family.name), + name, family: family.name, suffix: suffix(domain), plaintext: plaintext(family).map(|(kind, _)| kind), @@ -158,8 +270,100 @@ fn row(family: &'static DomainFamily, domain: &'static Domain) -> Row { ), indexes, query, - reason: target_gap(family, domain), + projections: if !is_stevec && v4_optional_query_terms(domain) { + Term::payload_terms(domain.terms) + .into_iter() + .map(|term| { + ( + index_key(term), + query_projection_ident(family, domain, term), + ) + }) + .collect() + } else { + Vec::new() + }, + reason, + answers, + } +} + +/// The typed query chain's impls for one type that answers an operator: +/// its query twin's assembly from the one term it carries, `QueryTerm` on +/// the twin when an exact operator returns it as is, and one `op::Answers` +/// per operator, through Stack Encrypt's index for it, sent as the twin. +fn answer_impls(r: &Row) -> TokenStream { + let module = format_ident!("{}", r.family); + let ty = format_ident!("{}", r.name); + let (twin_name, twin_domain) = r + .query + .as_ref() + .expect("a type that answers an operator has a query twin"); + let twin = format_ident!("{twin_name}"); + let (_, rust) = plaintext_of(r.family).expect("a producible family has a specified plaintext"); + let source: syn::Path = syn::parse_str(rust).expect("a family's Rust plaintext is a type path"); + // `answers_for_indexes` admits one index, so every operator's term is + // the same. + let term = r.answers[0].term(); + let field = format_ident!("{}", term.json_key()); + let newtype = format_ident!("{}", term.binding_newtype()); + let assemble_doc = format!( + " `{twin_name}` from the column it queries and its `{}` term, at version 4.", + term.json_key() + ); + let labelled_doc = format!( + " `{}` is sealed under the column a two-segment label names, so its plan is \ + queried under the plan's label.", + r.name + ); + let mut out = quote! { + #[doc = #labelled_doc] + impl op::LabelledTarget for super::#module::#ty {} + #[doc = #assemble_doc] + impl QueryTwin<#newtype> for super::#module::#twin { + type Plaintext = #source; + fn assemble(i: Identifier, #field: #newtype) -> Self { + Self { v: SchemaVersion::default(), i, #field } + } + } + }; + for answer in &r.answers { + let name = &r.name; + let (op, via, doc) = match answer { + Answer::Eq => { + let term_doc = + format!(" `{twin_name}` is exact: the query term of `{name}`'s `op::Eq`."); + out.extend(quote! { + #[doc = #term_doc] + impl op::QueryTerm for super::#module::#twin {} + }); + ( + quote!(op::Eq), + quote!(TwinIndex), + format!( + " `{name}` answers `op::Eq` through its equality index. The query \ + term is `{twin_name}`, bound as `{twin_domain}`." + ), + ) + } + Answer::Matches => ( + quote!(op::Matches), + quote!(TwinIndex, super::#module::#twin>), + format!( + " `{name}` answers `op::Matches` through its match index, under \ + `EqlMatch`. The query term is a `MatchTwin<{twin_name}>`: the twin, \ + bound as `{twin_domain}`, and the tokens that verify a row." + ), + ), + }; + out.extend(quote! { + #[doc = #doc] + impl op::Answers<#op, #source> for super::#module::#ty { + type Via = #via; + } + }); } + out } fn option_str(value: Option<&str>) -> TokenStream { @@ -237,14 +441,45 @@ fn render_targets_from(rows: &[Row]) -> String { .collect(); // A producible type with no query twin (a storage-only domain) gets no // query arm: the fall-through refuses it as answering no query. + // + // A twin whose terms are optional is built through the projection of + // the index that answers the query's operator, and carries that term + // alone; any other index is one the type does not carry, which + // `encryption::targets::query` refuses before dispatching here. let query_arms: TokenStream = producible .iter() .filter_map(|r| { let (query, _) = r.query.as_ref()?; let (name, ty, source) = arm(r, query); - Some(quote! { #name => run_target::<#ty, #source, K>(#name, keyset, column, plaintext), }) + if r.projections.is_empty() { + return Some( + quote! { #name => run_target::<#ty, #source, K>(#name, keyset, column, plaintext), }, + ); + } + let by_index: TokenStream = r + .projections + .iter() + .map(|(key, projection)| { + let (_, projection, _) = arm(r, projection); + quote! { + #key => run_projected::<#projection, #ty, #source, K>(#name, keyset, column, plaintext), + } + }) + .collect(); + Some(quote! { + #name => match index.key() { + #by_index + _ => Err(no_index(#name, index)), + }, + }) }) .collect(); + let projected = producible.iter().any(|r| !r.projections.is_empty()); + let index_param = if projected { + quote!(index) + } else { + quote!(_index) + }; // One `use` per plaintext a producible type names by path (`String` is // in the prelude), so the arms stay short enough to read. let mut imports: Vec<&str> = producible @@ -261,10 +496,10 @@ fn render_targets_from(rows: &[Row]) -> String { quote!(use #path;) }) .collect(); - let helpers = if producible.is_empty() { - quote!() - } else { - quote!(open_target, run_target,) + let helpers = match (producible.is_empty(), projected) { + (true, _) => quote!(), + (false, false) => quote!(open_target, run_target,), + (false, true) => quote!(no_index, open_target, run_projected, run_target,), }; let mod_doc = " The EQL types a Stack Encrypt data plan may name as a field target — \ @@ -279,6 +514,7 @@ fn render_targets_from(rows: &[Row]) -> String { #![doc = #mod_doc] use stack_encrypt::dynamic::ValueFrame; + use stack_encrypt::target::IndexSpec; #imports use crate::encryption::targets::{#helpers refuse, refuse_query, Opener, Target, TargetError}; @@ -325,8 +561,11 @@ fn render_targets_from(rows: &[Row]) -> String { /// Run the named type's query twin for one plaintext, or refuse the /// name: as `encrypt_named` does, or as answering no query /// (`TargetError::NoQuery`) for a producible type with no twin. + /// `index` answers the query's operator; a twin whose terms are + /// optional carries that index's term alone. pub(crate) fn query_named<'a, K: 'static>( name: &str, + #index_param: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, column: NonEmpty, plaintext: ValueFrame, @@ -341,6 +580,93 @@ fn render_targets_from(rows: &[Row]) -> String { format_rs(file) } +/// Render the generated `crates/eql-bindings/src/v4/answers.rs`. +pub fn render_answers_rs() -> String { + render_answers_from(&rows()) +} + +/// Render the typed query chain's impls from the given rows. A file of its +/// own, beside `targets.rs` rather than in it: it needs only the +/// `stack-encrypt` feature, so a Rust program queries a v4 type without +/// Stack Encrypt's `dynamic` layer, which `targets.rs` (the `targets` +/// feature) is built on. +fn render_answers_from(rows: &[Row]) -> String { + // The typed query chain: per type that answers an operator, its twin's + // assembly from the one term it carries, and one `op::Answers` impl per + // operator, through Stack Encrypt's index for it, sent as the twin. + let answering: Vec<&Row> = rows.iter().filter(|r| !r.answers.is_empty()).collect(); + let answer_impls: TokenStream = answering.iter().map(|r| answer_impls(r)).collect(); + let answer_entries: TokenStream = answering + .iter() + .map(|r| { + let name = &r.name; + let ops = r.answers.iter().map(|answer| answer.name()); + quote!((#name, &[#(#ops),*]),) + }) + .collect(); + let answers_any = |wanted: Answer| answering.iter().any(|r| r.answers.contains(&wanted)); + let mut answer_imports = TokenStream::new(); + if !answering.is_empty() { + let mut newtypes: Vec<&str> = answering + .iter() + .flat_map(|r| { + r.answers + .iter() + .map(|answer| answer.term().binding_newtype()) + }) + .collect(); + newtypes.sort_unstable(); + newtypes.dedup(); + let newtypes = newtypes.iter().map(|n| format_ident!("{n}")); + answer_imports.extend(quote! { + use crate::encryption::query::{QueryTwin, TwinIndex}; + use crate::Identifier; + use crate::v4::terms::{#(#newtypes),*}; + use crate::v4::SchemaVersion; + use stack_encrypt::op; + }); + } + if answers_any(Answer::Eq) { + answer_imports.extend(quote!( + use stack_encrypt::Equality; + )); + } + if answers_any(Answer::Matches) { + answer_imports.extend(quote! { + use crate::encryption::EqlMatch; + use stack_encrypt::Match; + }); + } + + let mod_doc = " How Stack Encrypt's typed query chain queries a producible EQL v4 \ + type: per type that answers an operator, its `op::Answers` impls, \ + its `op::LabelledTarget` impl and its query twin's assembly \ + (`crate::encryption::query`). Generated from the catalog, by the \ + rule the data path answers a field that names the type with."; + + let file = quote! { + #![doc = #mod_doc] + + #answer_imports + + /// The operators each EQL type answers in Stack Encrypt's typed + /// query chain (`op::Answers`), by the wire names + /// `stack_encrypt::dynamic::OpSpec` gives them, in catalog order: + /// one row per type that answers any. Generated from the rule the + /// impls below are, so the encryption tests can hold the impls to + /// the operators the data path answers for each type. Not part of + /// the supported API. + #[doc(hidden)] + pub const ANSWERS: &[(&str, &[&str])] = &[ + #answer_entries + ]; + + #answer_impls + }; + + format_rs(file) +} + #[cfg(test)] mod tests { use super::*; @@ -391,19 +717,18 @@ mod tests { target_gap(family, family.domain_by_name(name).expect("domain")) }; let text = eql_domains::TEXT; - for producible in ["", "eq", "match", "ord", "ord_ope"] { + for producible in [ + "", + "eq", + "match", + "ord", + "ord_ope", + "ord_ore", + "search", + "search_ore", + ] { assert!(gap(&text, producible).is_none(), "text {producible:?}"); } - assert!( - gap(&text, "search").unwrap().contains("match term"), - "TextSearch waits on how a query leaves out its match term" - ); - for block_ore in ["ord_ore", "search_ore"] { - assert!( - gap(&text, block_ore).unwrap().contains("block ORE"), - "{block_ore}" - ); - } assert!(gap(&text, "match").is_none(), "TextMatch is producible"); let family = |name: &str| { eql_domains::CATALOG @@ -414,20 +739,16 @@ mod tests { for family_name in ["integer", "smallint", "bigint", "real", "double"] { let family = family(family_name); assert!(gap(family, "").is_none(), "{family_name}"); - for producible in ["eq", "ord", "ord_ope"] { + for producible in ["eq", "ord", "ord_ope", "ord_ore"] { assert!( gap(family, producible).is_none(), "{family_name} {producible}" ); } - assert!( - gap(family, "ord_ore").unwrap().contains("block ORE"), - "{family_name}" - ); } // A date, a timestamp and a decimal are stored and compared for - // equality; every ordered domain of theirs waits for an order term, - // block ORE included, which waits for that first. + // equality; every ordered domain of theirs, block ORE included, + // waits for an order term. for family_name in ["date", "timestamp", "numeric"] { let family = family(family_name); assert!(gap(family, "").is_none(), "{family_name}"); @@ -450,7 +771,7 @@ mod tests { // a plan field spells an index by exactly these. assert_eq!(index_key(Term::Hm), "eq"); assert_eq!(index_key(Term::Bloom), "match"); - assert_eq!(index_key(Term::Ore), "ore"); + assert_eq!(index_key(Term::Ore), "ore_block"); assert_eq!(index_key(Term::Ope), "ope"); } @@ -523,29 +844,232 @@ mod tests { }; assert_eq!(count("run_target", &name), 1, "{name}: one encrypt arm"); assert_eq!(count("open_target", &name), 1, "{name}: one decrypt arm"); + let projected = v4_optional_query_terms(domain); assert_eq!( count("run_target", &format!("{name}Query")), - usize::from(!domain.terms.is_empty()), + usize::from(!domain.terms.is_empty() && !projected), "{name}: a query arm exactly when it has a query twin" ); + if projected { + // One arm per index, each through the projection for it. + for term in Term::payload_terms(domain.terms) { + let projection = query_projection_ident(family, domain, term); + let needle = format!( + "\"{}\"=>run_projected::(\"{name}\",", + index_key(term) + ); + // rustfmt leaves a trailing comma in a broken turbofish. + let unbroken = tight.replace(",>", ">"); + assert_eq!(unbroken.matches(&needle).count(), 1, "{needle}"); + } + } } // No unproducible type has an arm: the only `=>` arms are the - // producible ones and the three fall-throughs. - let with_twin = ENCRYPTION_DOMAINS + // producible ones, one per index of a projected twin, and the + // fall-throughs. + let (with_twin, by_index) = ENCRYPTION_DOMAINS .iter() - .filter(|(f, d)| { + .map(|(f, d)| { let family = eql_domains::CATALOG.iter().find(|x| x.name == *f).unwrap(); - !family.domain_by_name(d).unwrap().terms.is_empty() + family.domain_by_name(d).unwrap() }) - .count(); + .filter(|domain| !domain.terms.is_empty()) + .fold((0, 0), |(twins, by_index), domain| { + let indexes = if v4_optional_query_terms(domain) { + Term::payload_terms(domain.terms).len() + } else { + 0 + }; + (twins + 1, by_index + indexes) + }); + assert!(by_index > 0, "TextSearch's query is projected"); let arms = out.matches("\" => ").count(); assert_eq!( arms, - ENCRYPTION_DOMAINS.len() * 2 + with_twin, + ENCRYPTION_DOMAINS.len() * 2 + with_twin + by_index, "arms: {out}" ); assert_eq!(out.matches("_ => Err(refuse(name))").count(), 2); assert_eq!(out.matches("_ => Err(refuse_query(name))").count(), 1); + assert_eq!( + out.matches("_ => Err(no_index(\"TextSearch\", index))") + .count(), + 1 + ); + } + + /// A type answers operators in the typed query chain exactly when the + /// engine produces it and it has a query twin, and then by the engine's + /// rule over its indexes: `Eq` for the equality index, `Matches` for + /// the match index. Spelled out per type, so the table reads without + /// the rule. + #[test] + fn a_producible_type_with_a_twin_answers_what_its_indexes_answer() { + for row in rows() { + let answers: Vec<&str> = row.answers.iter().map(|a| a.name()).collect(); + if row.reason.is_some() || row.query.is_none() { + assert!(answers.is_empty(), "{}: answers nothing", row.name); + continue; + } + if let Some(gap) = typed_gap(&row.indexes) { + assert!(answers.is_empty(), "{}: {gap}", row.name); + continue; + } + let expected: Vec<&str> = row + .indexes + .iter() + .map(|key| match *key { + "eq" => "eq", + "match" => "matches", + other => panic!("{}: no answer for the {other} index", row.name), + }) + .collect(); + assert_eq!(answers, expected, "{}", row.name); + } + let answered: Vec<(String, Vec<&str>)> = rows() + .into_iter() + .filter(|r| !r.answers.is_empty()) + .map(|r| (r.name, r.answers.iter().map(|a| a.name()).collect())) + .collect(); + let spelled = |name: &str, ops: &[&'static str]| (name.to_owned(), ops.to_vec()); + assert_eq!( + answered, + vec![ + spelled("IntegerEq", &["eq"]), + spelled("SmallintEq", &["eq"]), + spelled("BigintEq", &["eq"]), + spelled("DateEq", &["eq"]), + spelled("TimestampEq", &["eq"]), + spelled("NumericEq", &["eq"]), + spelled("TextEq", &["eq"]), + spelled("TextMatch", &["matches"]), + spelled("RealEq", &["eq"]), + spelled("DoubleEq", &["eq"]), + ] + ); + } + + /// The rule over indexes: `eq` answers `Eq` and `match` answers + /// `Matches`; each operator's twin term is its index's. + #[test] + fn answers_for_indexes_maps_each_index_to_its_operator() { + assert_eq!(answers_for_indexes("T", &[]), Vec::::new()); + assert_eq!(answers_for_indexes("T", &["eq"]), vec![Answer::Eq]); + assert_eq!(answers_for_indexes("T", &["match"]), vec![Answer::Matches]); + assert_eq!(index_key(Answer::Eq.term()), "eq"); + assert_eq!(index_key(Answer::Matches.term()), "match"); + assert_eq!(Answer::Eq.name(), "eq"); + assert_eq!(Answer::Matches.name(), "matches"); + } + + /// A producible type whose twin carries an order index, or more than + /// one index, has no typed answer yet: the generator leaves it out of + /// the typed query chain and says why. + #[test] + fn an_order_index_or_several_indexes_answer_nothing_yet() { + for (name, indexes) in [ + ("IntegerOrd", &["ope"][..]), + ("IntegerOrdOre", &["ore"]), + ("TextOrd", &["eq", "ope"]), + ("TextSearch", &["eq", "ope", "match"]), + ] { + let gap = typed_gap(indexes).unwrap_or_else(|| panic!("{name} has a typed gap")); + assert!(gap.contains("order term"), "{name}: {gap}"); + assert_eq!( + answers_for_indexes(name, indexes), + Vec::::new(), + "{name}" + ); + } + let gap = typed_gap(&["eq", "match"]).expect("several terms are a gap"); + assert!(gap.contains("more than one term"), "{gap}"); + assert_eq!( + answers_for_indexes("T", &["eq", "match"]), + Vec::::new() + ); + assert_eq!(typed_gap(&[]), None); + assert_eq!(typed_gap(&["eq"]), None); + assert_eq!(typed_gap(&["match"]), None); + } + + #[test] + #[should_panic(expected = "no rule for the json index")] + fn an_unknown_index_stops_the_generator() { + answers_for_indexes("SteVecDocument", &["json"]); + } + + /// The rendered file implements `op::Answers` once per answered + /// operator, through the index the operator names, sent as the type's + /// twin; marks an exact twin a `QueryTerm`; and lists every answering + /// type in `ANSWERS`. + #[test] + fn rendered_answers_are_one_impl_per_operator_and_one_table_row_per_type() { + let tight: String = render_answers_rs().split_whitespace().collect(); + let mut answered = 0; + for row in rows() { + let module = row.family; + let name = &row.name; + let Some((twin, _)) = &row.query else { + continue; + }; + let answers_impl = |op: &str| format!("implop::Answers = row + .answers + .iter() + .map(|a| format!("\"{}\"", a.name())) + .collect(); + assert!( + tight.contains(&format!("(\"{name}\",&[{}]),", ops.join(","))), + "{name}: its ANSWERS row" + ); + for answer in &row.answers { + let (op, via) = match answer { + Answer::Eq => ( + "Eq", + format!("TwinIndex;"), + ), + Answer::Matches => ( + "Matches", + format!("TwinIndex,super::{module}::{twin}>;"), + ), + }; + assert!(tight.contains(&answers_impl(op)), "{name}: op::{op}"); + assert!(tight.contains(&via), "{name}: {via}"); + } + let exact = row.answers.contains(&Answer::Eq); + assert_eq!( + tight.contains(&format!("implop::QueryTermforsuper::{module}::{twin}{{}}")), + exact, + "{name}: QueryTerm exactly on an exact twin" + ); + assert_eq!( + tight + .matches(&format!("forsuper::{module}::{twin}{{typePlaintext=")) + .count(), + 1, + "{name}: one assembly" + ); + assert_eq!( + tight + .matches(&format!( + "implop::LabelledTargetforsuper::{module}::{name}{{}}" + )) + .count(), + 1, + "{name}: labelled" + ); + } + assert_eq!(answered, 10); + assert_eq!(tight.matches("implop::Answers<").count(), 10); + assert_eq!(tight.matches("implop::LabelledTarget").count(), 10); } /// A producible type with no query twin — the storage-only `Text` — diff --git a/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 b/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 index 9f3c60286..f4596433c 100644 --- a/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 @@ -1,7 +1,21 @@ --! @brief Index extractor for {{ dom }}. --! @param a {{ dom }} --! @return {{ e.ret }} +{%- if e.required %} +{{ v4_only }} +--! @throws When the operand carries no `{{ e.required.key }}` term, the term {{ e.required.operators }} compare. +{{ v4_end }} +{%- endif %} CREATE FUNCTION {{ schema }}.{{ e.extractor }}(a {{ dom }}) RETURNS {{ e.ret }} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +{%- if e.required %} +{{ v3_only }} AS $$ SELECT {{ internal_schema }}.{{ e.ctor }}(a::jsonb) $$; +{{ v3_end }} +{{ v4_only }} +AS $$ SELECT {{ internal_schema }}.{{ e.ctor }}({{ internal_schema }}.require_query_term(a::jsonb, '{{ e.required.domain }}', '{{ e.required.key }}', '{{ e.required.operators }}')) $$; +{{ v4_end }} +{%- else %} +AS $$ SELECT {{ internal_schema }}.{{ e.ctor }}(a::jsonb) $$; +{%- endif %} diff --git a/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 b/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 index 5c349bff7..e8903b167 100644 --- a/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 @@ -8,7 +8,7 @@ --! --! Runs after the DO block in src/v3/sem/ore_block_256/operator_class.sql, --! which ATTEMPTS to create the default btree operator class for ---! {{ internal_schema }}.ore_block_256 and skips it on insufficient_privilege +--! {{ internal_schema }}.{{ ore_block }} and skips it on insufficient_privilege --! (CREATE OPERATOR CLASS requires superuser; managed platforms — cloud --! Supabase and most hosted Postgres — run the installer as a non-superuser --! role). When the class was created, this file is a no-op. @@ -45,7 +45,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{ internal_schema }}.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{ internal_schema }}.{{ ore_block }}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 b/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 index eb93b0266..db992f483 100644 --- a/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 @@ -26,13 +26,35 @@ BEGIN {%- for k in d.keys %} AND VALUE ? '{{ k }}' {%- endfor %} + {%- if d.optional_term_keys %} +{{ v3_only }} + {%- for k in d.optional_term_keys %} + AND VALUE ? '{{ k }}' + {%- endfor %} +{{ v3_end }} +{{ v4_only }} + AND ({% for k in d.optional_term_keys %}{% if not loop.first %} OR {% endif %}VALUE ? '{{ k }}'{% endfor %}) +{{ v4_end }} + {%- endif %} {%- for k in d.forbidden_keys %} AND NOT (VALUE ? '{{ k }}') {%- endfor %} + {%- if d.optional_term_keys and d.nonempty_array_keys %} +{{ v3_only }} + {%- endif %} {%- for k in d.nonempty_array_keys %} AND jsonb_typeof(VALUE -> '{{ k }}') = 'array' AND jsonb_array_length(VALUE -> '{{ k }}') > 0 {%- endfor %} + {%- if d.optional_term_keys and d.nonempty_array_keys %} +{{ v3_end }} +{{ v4_only }} + {%- for k in d.nonempty_array_keys %} + AND (NOT (VALUE ? '{{ k }}') OR (jsonb_typeof(VALUE -> '{{ k }}') = 'array' + AND jsonb_array_length(VALUE -> '{{ k }}') > 0)) + {%- endfor %} +{{ v4_end }} + {%- endif %} AND VALUE->>'v' = '{{ eql_version }}' ); END IF; diff --git a/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 b/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 index a0fa8a0ac..1754139b6 100644 --- a/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 @@ -41,7 +41,7 @@ AS $$ SELECT {{ a.body }} $$; --! stop inlining. Inlined, the planner sees two comparisons on --! `{{ schema }}.{{ b.extractor }}(a)`, which a btree index on that expression --! serves as one range scan.{% if b.ore %} For this ORE range that index needs the ---! default btree operator class on {{ internal_schema }}.ore_block_256, which the +--! default btree operator class on {{ internal_schema }}.{{ ore_block }}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column ({{ b.ope_columns }}) diff --git a/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 b/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 index 82259afc1..4a4b87045 100644 --- a/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 @@ -59,7 +59,7 @@ END $$; {%- if ore_ranges %} --- Without the default btree operator class on {{ internal_schema }}.ore_block_256 +-- Without the default btree operator class on {{ internal_schema }}.{{ ore_block }} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{ internal_schema }}.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -72,7 +72,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{ internal_schema }}.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{ internal_schema }}.{{ ore_block }}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/crates/eql-codegen/tests/cli.rs b/packages/eql/crates/eql-codegen/tests/cli.rs index 5e9f6abd0..86de21ef8 100644 --- a/packages/eql/crates/eql-codegen/tests/cli.rs +++ b/packages/eql/crates/eql-codegen/tests/cli.rs @@ -35,7 +35,7 @@ fn tempdir() -> TempDir { /// the output-root override (test isolation) and never touches the committed /// `crates/eql-bindings/src/v3/*.rs`. The count is one file per scalar family /// plus the jsonb family's generated `jsonb_storage.rs`, `payload.rs`, -/// `query_payload.rs`, `targets.rs`, and `inventory.rs`. +/// `query_payload.rs`, `targets.rs`, `answers.rs` and `inventory.rs`. #[test] fn bindings_subcommand_succeeds_and_reports_count() { let out_root = tempdir(); @@ -51,11 +51,12 @@ fn bindings_subcommand_succeeds_and_reports_count() { ); let stdout = String::from_utf8_lossy(&out.stdout); // v3: scalar families + json_storage + payload + query_payload + - // inventory. v4: every family with a scalar domain + targets + inventory. + // inventory. v4: every family with a scalar domain + targets + answers + + // inventory. let expected = eql_domains::scalar_families().count() + 4 + eql_domains::families_with_scalar_domains().count() - + 2; + + 3; assert!( stdout.contains(&format!("bindings: ok ({expected} files)")), "expected 'bindings: ok ({expected} files)' in stdout, got:\n{stdout}" diff --git a/packages/eql/docs/upgrading/v3.1.md b/packages/eql/docs/upgrading/v3.1.md new file mode 100644 index 000000000..9c8eb257a --- /dev/null +++ b/packages/eql/docs/upgrading/v3.1.md @@ -0,0 +1,190 @@ +# Upgrading to EQL 3.1 + +`3.1.0` is a minor release. It changes two things in the `eql_v4` surface that +a caller can meet without changing any SQL: which terms a text search query +must carry, and which block ORE ciphertext the `_ord_ore` and +`text_search_ore` domains hold. **The `eql_v3` bundle is unchanged, byte for +byte.** If you install only `eql_v3`, nothing here applies to you. + +Both changes land through Stack Encrypt as well as through the SQL: with +`3.1.0`, Stack Encrypt produces `eql_v4_text_search` and every block ORE +type (`_ord_ore` and `text_search_ore`), so a value written by it carries the +terms this release's `eql_v4` expects. The notes below are for anyone who +builds `eql_v4` payloads by hand, or who stored `eql_v4` block ORE terms +before this release. + +## TL;DR + +1. An `eql_v4` text search query carries only the term its operator reads, + and an operator reading a term the query does not carry **raises** + (`invalid_parameter_value`) instead of matching nothing or everything — + [U-001](#u-001-a-text-search-query-carries-only-the-terms-its-operator-reads). +2. The `eql_v4` block ORE term is ore-rs's 6-bit scheme, compared by a new + comparator named `ore_block_v2`. It **refuses** the 8-bit terms the + previous `eql_v4` bundle compared; rows and queries holding them must be + written again through Stack Encrypt — + [U-002](#u-002-the-block-ore-term-is-ore-rss-6-bit-scheme). + +## Compatibility + +| Surface | Status | +| --- | --- | +| `eql_v3`: every domain, operator, extractor and term | **Unchanged.** The v3 bundle is byte for byte what 3.0.6 shipped; its `ob` is still the 8-bit block ORE under `ore_block_256`. | +| `eql_v4.query_text_search`, `eql_v4.query_text_search_ore` | **Relaxed CHECK.** At least one of the terms (`hm`; `op` or `ob`; `bf`) is required, where all three were. See U-001. | +| `=`, `<>`, `<`, `<=`, `>`, `>=`, `@@` on `public.eql_v4_text_search` / `public.eql_v4_text_search_ore` | **Tightened error semantics.** A query without the term the operator reads raises `invalid_parameter_value` wherever the operand is read. See U-001. | +| Stored `public.eql_v4_text_search` / `public.eql_v4_text_search_ore` values | **Unchanged.** A stored value still carries every term. Text too short for a match term stores the empty filter, `"bf": []`, which `=` and `<` find and no `@@` query does. | +| `public.eql_v4_*_ord_ore`, `public.eql_v4_text_search_ore`, `eql_v4.query_*_ord_ore`, `eql_v4.query_*_range_ore` | **Term format changed.** The `ob` term is one ore-rs 6-bit block ORE ciphertext. The previous 8-bit layout is refused, not misread. Re-encryption of those columns is required where such rows exist. See U-002. | +| `eql_v4_internal.ore_block_256`, `compare_ore_block_256_term` | **Renamed** in the v4 bundle to `eql_v4_internal.ore_block_v2` and `compare_ore_block_v2_term`, with a new body. Only the installer and the operator class name them. | +| `eql_v4` OPE (`op`) terms, equality (`hm`) terms, ciphertexts (`c`) | **Unchanged** by this release. | +| Postgres 14, 15, 16, 17 | Supported. | +| Supabase / managed Postgres (no superuser) | As before: the block ORE domains need the operator class the installer creates, so a non-superuser install disables them ([U-003 in the 3.0 guide](./v3.0.md#u-003-non-superuser-installs-disable-the-ore-backed-domains)). U-002 only concerns installs where they are enabled. | + +## Upgrade notes + +### U-001: A text search query carries only the terms its operator reads + +**What changed.** `eql_v4.query_text_search` and +`eql_v4.query_text_search_ore` required every term of the type: `hm` for +`=`, `op` (or `ob`) for the orderings and `bf` for `@@`. They now require at +least one. Each operator reads one term, and the extractor it reads through +raises when the query does not carry it, carries it as JSON `null`, or +carries an empty array: + +```text +ERROR: query_text_search value has no "hm" term, the term = <> compare +SQLSTATE: 22023 (invalid_parameter_value) +``` + +The message names the domain, the term and the operators that read it, +never the operand. The raise happens wherever the operand is read, so a +statement whose predicate is never reached (a generic plan over an empty +table, say) returns no row and no error. + +**Why.** Text shorter than one 3-gram has no match term, so with every term +required a search column could never be asked `name = $1` or `name < $1` for +such text, although both have an answer. And a term that is present but +unreadable must not become a silent miss: before, a missing term read as +`NULL`, which compares equal to nothing and matches no row. + +**Who is affected.** A caller that builds query operands by hand, or that +checks them against the domain CHECK. Stack Encrypt builds the query for the +operator's index and carries that term alone; nothing changes for its +callers. Stored values are untouched: a stored search value still carries +every term, and a row stored before this release is found exactly as before. + +**What to do.** Nothing, for stored data. If you build a text search query +yourself, carry the term the operator reads; carrying the others as well is +still valid. Do not pass `null` or `[]` for a term you lack: omit it, and ask +only through operators whose term you carry. If you handled "no result" from +a query with a missing term, expect an error now. + +**Verification.** + +```sql +-- A query carrying one term is a valid operand (was refused by the CHECK): +SELECT '{"v": 4, "i": {"t": "users", "c": "name"}, "hm": "00"}' + ::jsonb::eql_v4.query_text_search; + +-- A query carrying no term is still refused by the CHECK (23514): +SELECT '{"v": 4, "i": {"t": "users", "c": "name"}}' + ::jsonb::eql_v4.query_text_search; +``` + +Then, on a table with at least one row, `WHERE name < $1` with the `hm`-only +operand above must fail with SQLSTATE `22023` and the message naming `op`. + +### U-002: The block ORE term is ore-rs's 6-bit scheme + +**What changed.** The `ob` term of every `eql_v4` block ORE domain is now +one ciphertext of ore-rs's 6-bit block ORE: the fixed-width scheme for +numbers, the chained scheme for text, which pads the text to a power-of-two +length bucket of at least 16 bytes. The v4 bundle names its block ORE objects +`ore_block_v2` (`eql_v4_internal.ore_block_v2_term`, +`eql_v4_internal.compare_ore_block_v2_term`) and compares them with a new +comparator that checks each term's shape before comparing, then runs the +same number of statements whatever the terms hold. It raises on: + +- a term that is not a 6-bit block ORE ciphertext, the previous 8-bit layout + included: `Malformed ORE block term: N bytes`; +- two terms of different schemes (fixed-width against chained): `ORE block + terms of different schemes`; +- two chained terms padded under different policies: `ORE block terms padded + under different policies`. + +The `eql_v3` bundle keeps the 8-bit scheme under `ore_block_256`, unchanged. + +**Why.** The 6-bit scheme is what Stack Encrypt derives, and with it Stack +Encrypt produces the block ORE types (`IntegerOrdOre`, `SmallintOrdOre`, +`BigintOrdOre`, `RealOrdOre`, `DoubleOrdOre`, `TextOrdOre`, +`TextSearchOre`) for the first time. One comparator per scheme, with the +scheme in the name, keeps a v3 term and a v4 term from ever being compared +by the other's rules. + +**Who is affected.** No Stack Encrypt release wrote an `eql_v4` block ORE +term before this one: the block ORE types were not producible. So a v4 +`_ord_ore` or `text_search_ore` row holding an 8-bit term exists only where +it was written by hand, or with cipherstash-client's ORE indexer, against +the earlier `eql_v4` bundle, which compared such terms. If you have such +rows, or query operands or `eql_v4.between` ranges built that way, they are +refused by this release's comparator with the first error above. They are +never ordered wrongly: a refused term is an error, not a result. + +**What to do.** Where such rows exist, write the affected columns again +through Stack Encrypt, which derives the 6-bit term for every block ORE +type, and build every block ORE query and range operand through it as well. +Do this before creating or rebuilding a functional index over +`eql_v4.ord_term_ore(col)`: an index build compares every row, so one +8-bit term in the table fails the build. Installs that never held an +`eql_v4` block ORE term, and installs whose block ORE domains are disabled +(no operator class), have nothing to do. + +**Verification.** + +```sql +-- The v4 bundle names its block ORE objects by the scheme: +SELECT p.proname +FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname = 'eql_v4_internal' AND p.proname LIKE '%ore_block%' +ORDER BY 1; -- ore_block_v2 names only; no ore_block_256 + +-- Every stored term in a block ORE column is one the comparator accepts: +-- this compares each row's term with itself and raises on a term it refuses. +SELECT count(*) FROM users WHERE age <= age; -- equals count(*), no error +``` + +## Verification checklist + +- [ ] The `eql_v4` install reports `3.1.0` (`SELECT obj_description('eql_v4'::regnamespace, 'pg_namespace')`). +- [ ] A one-term `eql_v4.query_text_search` operand casts (U-001). +- [ ] `eql_v4_internal` lists `ore_block_v2` objects and no `ore_block_256` (U-002). +- [ ] Every `_ord_ore` / `text_search_ore` column passes the self-comparison scan (U-002). +- [ ] Grants and functional indexes have been re-applied after the install + ([U-002 in the 3.0.5 guide](./v3.0.5.md#u-002-grants-and-dependent-objects-do-not-survive-an-install)). + +## Rollback + +Installing the 3.0.6 bundle reverses both notes at the SQL, and neither +reversal misreads data: + +- U-001: the 3.0.6 CHECK requires every term again, so a one-term query is + refused by the domain (SQLSTATE `23514`) rather than answered. +- U-002: the 3.0.6 v4 comparator expects the 8-bit layout, so a 6-bit term + written under 3.1.0 is refused as malformed, exactly as the 8-bit terms are + under 3.1.0. Rolling back a database that holds 6-bit terms therefore + means rewriting those columns too, with a Stack Encrypt that derives the + 8-bit term, and no release does. Decide before writing block ORE rows under + 3.1.0 whether a rollback must stay possible; if it must, hold off on the + block ORE domains and order through the `_ord` / `_ord_ope` domains, whose + terms this release does not change. + +Any install, in either direction, re-runs `DROP SCHEMA … CASCADE`, so grants +and dependent objects need re-applying afterwards. + +## See also + +- [Upgrading to EQL 3.0.5](./v3.0.5.md) — grants and dependent objects do + not survive an install. +- [Upgrading to EQL 3.0](./v3.0.md) — the 3.0 line, including why a + non-superuser install disables the block ORE domains. +- [Database indexes](../reference/database-indexes.md) — the index recipes, + and which need the ORE operator class. diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index 128ab2fcb..9f1995443 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -322,7 +322,7 @@ cargo clippy --locked -p eql-bindings --features stack-encrypt --all-targets -- """ [tasks."test:encryption:postgres"] -description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the PostgreSQL tests against it: stack-encrypt values (fresh encryption, fake KMS), and the range query domains and eql_v4.between" +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the PostgreSQL tests against it: stack-encrypt values (fresh encryption, fake KMS), the range query domains and eql_v4.between, and the JSON tree family" # `build` writes release/cipherstash-encrypt-v4.sql from the SQL in the tree. The # copy under crates/eql-bindings/sql is the RELEASED bundle, rewritten only by # the lockstep version bump, so a test that installs it cannot see a SQL edit @@ -340,7 +340,7 @@ psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -f release/cipherstash-encrypt- # `--ignored` selects by attribute, and cargo exits 0 on an empty selection, so # a test that loses its `#[ignore]` or is renamed would leave this task passing # on nothing. List what the filter selects before running it. -for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range; do +for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range v4_text_search v4_ore_block json_tree; do listing=$(cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --list | sed -n 's/: test$//p') [ -n "$listing" ] || { echo "error: '--ignored' selects no test in $suite; the PostgreSQL test has gone missing." >&2; exit 1; } cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored diff --git a/packages/eql/src/v3/common.sql b/packages/eql/src/v3/common.sql index 56c0edc33..4097ec4f2 100644 --- a/packages/eql/src/v3/common.sql +++ b/packages/eql/src/v3/common.sql @@ -19,7 +19,7 @@ --! @note Returns NULL if input is JSON null --! @note Each array element is hex-decoded to bytea --! @note plpgsql, not `LANGUAGE sql` (issue #353). This helper's ONLY caller ---! chain is `ore_block_256(val)` -> `jsonb_array_to_ore_block_256(val)` — +--! chain is `{{ore_block}}(val)` -> `jsonb_array_to_{{ore_block}}(val)` — --! both reached exclusively from plpgsql and btree operator-class support --! contexts, where SQL functions can NEVER be inlined and instead pay the --! per-call SQL-function executor (measured 3.5x the per-call cost of the @@ -49,7 +49,7 @@ $$ LANGUAGE plpgsql; --! @internal Keep the inline-critical marker so the post-install --! pin_search_path pass leaves this unpinned: a `SET search_path` clause on a --! plpgsql function forces per-call configuration switching — measurable on a ---! helper invoked per compared value in the ore_block_256 opclass hot path. +--! helper invoked per compared value in the {{ore_block}} opclass hot path. --! It takes a bare `jsonb` arg (not a jsonb-backed encrypted DOMAIN), so the --! structural skip in tasks/pin_search_path_v3.sql does not recognise it; --! this marker is the documented manual opt-in. diff --git a/packages/eql/src/v3/json_tree/blockers.sql b/packages/eql/src/v3/json_tree/blockers.sql new file mode 100644 index 000000000..bff60387e --- /dev/null +++ b/packages/eql/src/v3/json_tree/blockers.sql @@ -0,0 +1,1118 @@ +-- ONLY EQL VERSION: 4 +-- REQUIRE: src/v3/schema.sql +-- REQUIRE: src/v3/json_tree/types.sql +-- REQUIRE: src/v3/scalars/functions.sql + +--! @file v3/json_tree/blockers.sql +--! @brief Native-jsonb firewall for public.{{prefix}}_json_tree. +--! +--! public.{{prefix}}_json_tree SUPPORTS @> <@ -> ->> (see operators.sql), and +--! public.{{prefix}}_json_tree_entry the ordered comparisons < <= > >= against an +--! ordering operand or another entry. Entry equality = <> is blocked: an +--! entry's ordering term is not an exact equality representation, and exact +--! equality at a path is containment on a value selector. An entry against +--! plain jsonb is blocked for every comparison, in both directions, so an +--! ordering operand that lost its `{{prefix}}.query_json_tree_leaf` cast +--! raises instead of comparing ciphertext. Root-document comparisons are +--! blocked too. `ORDER BY col -> 'sel'` cannot be blocked without an operator +--! class, which needs a superuser: it sorts by jsonb's ordering of the entry, +--! so order by `{{prefix}}.ord_term(col -> 'sel')` instead. Every OTHER native jsonb operator reachable +--! via domain fallback against the base type jsonb is BLOCKED here, so an +--! encrypted column can never silently route to plaintext-jsonb semantics. +--! +--! Each blocker is LANGUAGE plpgsql (NEVER STRICT: a STRICT blocker would let +--! PostgreSQL skip the body and return NULL on a NULL argument, bypassing the +--! exception) and delegates to the shared {{prefix}}_internal.encrypted_domain_unsupported_* +--! helpers. Each blocker's RETURNS type matches the native operator it shadows +--! so a composed expression resolves and the body raises 'operator not +--! supported', rather than failing earlier on a boolean intermediate. + +--! @brief Blocker: ? (key/element exists). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists(a public.{{prefix}}_json_tree, b text) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '?'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR ? ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_exists, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text +); + +--! @brief Blocker: ?| (any key exists). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text[] Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists_any(a public.{{prefix}}_json_tree, b text[]) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '?|'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR ?| ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_exists_any, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text[] +); + +--! @brief Blocker: ?& (all keys exist). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text[] Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists_all(a public.{{prefix}}_json_tree, b text[]) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '?&'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR ?& ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_exists_all, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text[] +); + +--! @brief Blocker: @? (jsonpath exists). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonpath Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonpath_exists(a public.{{prefix}}_json_tree, b jsonpath) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@?'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR @? ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonpath_exists, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonpath +); + +--! @brief Blocker: @@ (jsonpath predicate). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonpath Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonpath_match(a public.{{prefix}}_json_tree, b jsonpath) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@@'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR @@ ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonpath_match, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonpath +); + +--! @brief Blocker: #> (path extract, native returns jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text[] Native RHS operand. +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_path_extract(a public.{{prefix}}_json_tree, b text[]) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '#>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR #> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_path_extract, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text[] +); + +--! @brief Blocker: #>> (path extract as text). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text[] Native RHS operand. +--! @return text Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_path_extract_text(a public.{{prefix}}_json_tree, b text[]) +RETURNS text +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_text('public.{{prefix}}_json_tree', '#>>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR #>> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_path_extract_text, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text[] +); + +--! @brief Blocker: - (delete key, text RHS). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text Native RHS operand. +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_text(a public.{{prefix}}_json_tree, b text) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '-'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR - ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_text, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text +); + +--! @brief Blocker: - (delete index, integer RHS). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b integer Native RHS operand. +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_int(a public.{{prefix}}_json_tree, b integer) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '-'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR - ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_int, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = integer +); + +--! @brief Blocker: - (delete keys, text[] RHS). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text[] Native RHS operand. +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_array(a public.{{prefix}}_json_tree, b text[]) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '-'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR - ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_array, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text[] +); + +--! @brief Blocker: #- (delete at path). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b text[] Native RHS operand. +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_path(a public.{{prefix}}_json_tree, b text[]) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '#-'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR #- ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_path, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = text[] +); + +--! @brief Blocker: || (concatenate, encrypted on the left). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_concat(a public.{{prefix}}_json_tree, b jsonb) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '||'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR || ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_concat, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: || (concatenate, encrypted on the right). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_concat_rhs(a jsonb, b public.{{prefix}}_json_tree) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '||'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR || ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_concat_rhs, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: -> (element by position): positions are tokens, not selectors. +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b integer Native RHS operand. +--! @return jsonb Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_select_int(a public.{{prefix}}_json_tree, b integer) +RETURNS jsonb +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '->'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR -> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_select_int, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = integer +); + +--! @brief Blocker: ->> (element by position, as text). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b integer Native RHS operand. +--! @return text Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_select_int_text(a public.{{prefix}}_json_tree, b integer) +RETURNS text +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_text('public.{{prefix}}_json_tree', '->>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR ->> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_select_int_text, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = integer +); + +--! @brief Blocker: root document = (document, document). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_eq_doc_doc, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document = (document, jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_eq_doc_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: root document = (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_eq_jsonb_doc, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document <> (document, document). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_neq_doc_doc, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document <> (document, jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_neq_doc_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: root document <> (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_neq_jsonb_doc, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document < (document, document). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR < ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lt_doc_doc, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document < (document, jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR < ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lt_doc_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: root document < (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR < ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lt_jsonb_doc, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document <= (document, document). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lte_doc_doc, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document <= (document, jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lte_doc_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: root document <= (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lte_jsonb_doc, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document > (document, document). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR > ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gt_doc_doc, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document > (document, jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR > ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gt_doc_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: root document > (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR > ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gt_jsonb_doc, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document >= (document, document). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gte_doc_doc, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: root document >= (document, jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gte_doc_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: root document >= (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gte_jsonb_doc, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: = between two extracted entries (no exact equality term). +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_eq(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_eq, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: <> between two extracted entries. +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_neq(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_neq, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: < (entry, jsonb): an ordering operand is typed. +--! +--! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! entry. Against plain jsonb, the native operator would compare the entry's +--! ciphertext object, which is what a query that forgets the operand's cast +--! would silently get. +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_lt_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR < ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_lt_jsonb, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = jsonb +); + +--! @brief Blocker: < (jsonb, entry). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_lt_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR < ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_lt_entry, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: <= (entry, jsonb): an ordering operand is typed. +--! +--! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! entry. Against plain jsonb, the native operator would compare the entry's +--! ciphertext object, which is what a query that forgets the operand's cast +--! would silently get. +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_lte_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_lte_jsonb, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = jsonb +); + +--! @brief Blocker: <= (jsonb, entry). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_lte_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_lte_entry, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: > (entry, jsonb): an ordering operand is typed. +--! +--! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! entry. Against plain jsonb, the native operator would compare the entry's +--! ciphertext object, which is what a query that forgets the operand's cast +--! would silently get. +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_gt_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR > ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_gt_jsonb, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = jsonb +); + +--! @brief Blocker: > (jsonb, entry). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_gt_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR > ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_gt_entry, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: >= (entry, jsonb): an ordering operand is typed. +--! +--! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! entry. Against plain jsonb, the native operator would compare the entry's +--! ciphertext object, which is what a query that forgets the operand's cast +--! would silently get. +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_gte_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_gte_jsonb, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = jsonb +); + +--! @brief Blocker: >= (jsonb, entry). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_gte_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >= ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_gte_entry, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: = (entry, jsonb): an ordering operand is typed. +--! +--! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! entry. Against plain jsonb, the native operator would compare the entry's +--! ciphertext object, which is what a query that forgets the operand's cast +--! would silently get. +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_eq_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_eq_jsonb, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = jsonb +); + +--! @brief Blocker: = (jsonb, entry). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_eq_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '='); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_eq_entry, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: <> (entry, jsonb): an ordering operand is typed. +--! +--! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! entry. Against plain jsonb, the native operator would compare the entry's +--! ciphertext object, which is what a query that forgets the operand's cast +--! would silently get. +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_neq_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_neq_jsonb, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = jsonb +); + +--! @brief Blocker: <> (jsonb, entry). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_neq_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_neq_entry, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: @> (document, entry): an entry is not a needle. +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_entry(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree_entry) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR @> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_contains_entry, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = public.{{prefix}}_json_tree_entry +); + +--! @brief Blocker: <@ (entry, document). +--! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_entry(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<@'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <@ ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_contained_by_entry, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: @> (document, jsonb): a needle must be typed. +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR @> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_contains_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: @> (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_jsonb_lhs(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@>'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR @> ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_contains_jsonb_lhs, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); + +--! @brief Blocker: <@ (document, jsonb). +--! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param b jsonb Native RHS operand. +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_jsonb(a public.{{prefix}}_json_tree, b jsonb) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<@'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <@ ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_contained_by_jsonb, + LEFTARG = public.{{prefix}}_json_tree, + RIGHTARG = jsonb +); + +--! @brief Blocker: <@ (jsonb, document). +--! @param a jsonb Native LHS operand. +--! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @return boolean Never returns; always raises 'operator not supported'. +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_jsonb_lhs(a jsonb, b public.{{prefix}}_json_tree) +RETURNS boolean +IMMUTABLE PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<@'); +END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <@ ( + FUNCTION = {{prefix}}_internal.json_tree_blocked_contained_by_jsonb_lhs, + LEFTARG = jsonb, + RIGHTARG = public.{{prefix}}_json_tree +); diff --git a/packages/eql/src/v3/json_tree/functions.sql b/packages/eql/src/v3/json_tree/functions.sql new file mode 100644 index 000000000..7c7dca98e --- /dev/null +++ b/packages/eql/src/v3/json_tree/functions.sql @@ -0,0 +1,544 @@ +-- ONLY EQL VERSION: 4 +-- REQUIRE: src/v3/schema.sql +-- REQUIRE: src/v3/json_tree/types.sql +-- REQUIRE: src/v3/sem/ope_cllw/types.sql +-- REQUIRE: src/v3/sem/ope_cllw/functions.sql + +--! @file v3/json_tree/functions.sql +--! @brief The JSON tree surface's functions: the chain primitives, the +--! containment projection, extraction by selector, the subtree walk +--! by token, and the ordering extractors. +--! +--! Everything here is core PostgreSQL: `sha256(bytea)`, `get_byte` / +--! `set_byte`, `decode` / `encode`, recursive CTEs and `jsonb` operators. No +--! extension, no superuser, so the surface installs wherever the rest of +--! {{prefix}} does. +--! +--! `selector` parameters are *encrypted-side* selectors, the hex the index +--! emits in `s`; `token` parameters are the hex of a chain token the client +--! derived from a path. Plaintext paths are never accepted. + +------------------------------------------------------------------------------ +-- Chain primitives ({{prefix}}_internal) +------------------------------------------------------------------------------ + +--! @brief The first sixteen bytes of SHA-256 over `x`. +--! @internal +--! @param x bytea The bytes to hash. +--! @return bytea Sixteen bytes. +CREATE FUNCTION {{prefix}}_internal.json_tree_h16(x bytea) + RETURNS bytea + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT substr(sha256(x), 1, 16) +$$; + +--! @brief XOR of two sixteen-byte strings. +--! @internal +--! @param a bytea Sixteen bytes. +--! @param b bytea Sixteen bytes. +--! @return bytea `a XOR b`, byte by byte. +CREATE FUNCTION {{prefix}}_internal.json_tree_xor16(a bytea, b bytea) + RETURNS bytea + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + r bytea := a; + i int; +BEGIN + FOR i IN 0..15 LOOP + r := set_byte(r, i, get_byte(a, i) # get_byte(b, i)); + END LOOP; + RETURN r; +END; +$$; + +--! @brief A 32-bit integer as four little-endian bytes. +--! @internal +--! @param n integer The integer, non-negative. +--! @return bytea Four bytes, least significant first. +CREATE FUNCTION {{prefix}}_internal.json_tree_le32(n integer) + RETURNS bytea + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT set_byte(set_byte(set_byte(set_byte('\x00000000'::bytea, + 0, n & 255), 1, (n >> 8) & 255), 2, (n >> 16) & 255), 3, (n >> 24) & 255) +$$; + +--! @brief The document's ZeroKMS `iv` from its header: the per-document salt +--! of every chain tag. +--! @internal +--! @param h text The document's header, base64. +--! @return bytea Sixteen bytes, at byte offset 22 of the header's frozen +--! layout (version, flags, epoch, keyset id, iv, tag). +CREATE FUNCTION {{prefix}}_internal.json_tree_iv(h text) + RETURNS bytea + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT substr(decode(h, 'base64'), 23, 16) +$$; + +--! @brief The parent tag a child at ordinal `o` under the node with token +--! `token` carries: `H("p" ‖ iv ‖ token ‖ LE32(o))`. +--! @internal +--! @param iv bytea The document's iv. +--! @param token bytea The parent's token, sixteen bytes. +--! @param o integer The child's ordinal. +--! @return text The tag, hex, as the entry's `p` spells it. +CREATE FUNCTION {{prefix}}_internal.json_tree_parent_tag(iv bytea, token bytea, o integer) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT encode({{prefix}}_internal.json_tree_h16('p'::bytea || iv || token || {{prefix}}_internal.json_tree_le32(o)), 'hex') +$$; + +--! @brief Unwrap a child's token from its entry, given its parent's token: +--! `w XOR H("w" ‖ iv ‖ token ‖ c)`. +--! @internal +--! @param iv bytea The document's iv. +--! @param token bytea The parent's token. +--! @param entry jsonb The child's entry, carrying `w` and `c`. +--! @return bytea The child's token, sixteen bytes. +CREATE FUNCTION {{prefix}}_internal.json_tree_unwrap(iv bytea, token bytea, entry jsonb) + RETURNS bytea + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}_internal.json_tree_xor16( + decode(entry ->> 'w', 'hex'), + {{prefix}}_internal.json_tree_h16('w'::bytea || iv || token || decode(entry ->> 'c', 'base64')) + ) +$$; + +--! @brief How many ordinals a walk tries under each node: one more than the +--! largest ordinal any entry of the document carries. +--! @internal +--! @param val jsonb The document. +--! @return integer The fan-out bound; `0` for a document with no chained +--! entry. +CREATE FUNCTION {{prefix}}_internal.json_tree_fanout(val jsonb) + RETURNS integer + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT COALESCE(MAX((elem ->> 'o')::integer), -1) + 1 + FROM jsonb_array_elements(val -> 'sv') AS elem +$$; + +------------------------------------------------------------------------------ +-- Envelope helpers +------------------------------------------------------------------------------ + +--! @brief The envelope metadata an extracted entry carries: `i`, `v`, `h`. +--! @param val jsonb The document. +--! @return jsonb `{i, v, h}`, with absent keys dropped. +CREATE FUNCTION {{prefix}}.json_tree_meta(val jsonb) + RETURNS jsonb + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT jsonb_strip_nulls( + jsonb_build_object('i', val -> 'i', 'v', val -> 'v', 'h', val -> 'h') + ) +$$; + +COMMENT ON FUNCTION {{prefix}}.json_tree_meta(jsonb) IS + 'eql-inline-critical: raw-jsonb envelope helper used by the JSON tree extractors; must stay inlinable (unpinned search_path)'; + +--! @brief The entry at ordinal `j` of the document, with the metadata grafted: +--! what every extractor returns per node. +--! @internal +--! @param val jsonb The document. +--! @param elem jsonb The entry, as it sits in `sv`. +--! @param j integer The entry's ordinal in `sv`, from zero. +--! @return public.{{prefix}}_json_tree_entry The entry with `i`, `v`, `h` and `j`. +CREATE FUNCTION {{prefix}}_internal.json_tree_graft(val jsonb, elem jsonb, j integer) + RETURNS public.{{prefix}}_json_tree_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT ({{prefix}}.json_tree_meta(val) || elem || jsonb_build_object('j', j))::public.{{prefix}}_json_tree_entry +$$; + +------------------------------------------------------------------------------ +-- Extraction by selector +------------------------------------------------------------------------------ + +--! @brief Every entry whose path selector is `selector`, as rows. +--! +--! Every element of an array shares its parent's selector, so a wildcard +--! path (`$.tags[*]`) returns one row per element. Each row opens on its +--! own: it carries the document's header `h` and its own ordinal `j`. +--! +--! @param val jsonb The document. +--! @param selector text The path selector, hex. +--! @return SETOF public.{{prefix}}_json_tree_entry The matching entries, in +--! document order. +CREATE FUNCTION {{prefix}}.json_tree_path_query(val jsonb, selector text) + RETURNS SETOF public.{{prefix}}_json_tree_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}_internal.json_tree_graft(val, elem, (ord - 1)::integer) + FROM jsonb_array_elements(val -> 'sv') WITH ORDINALITY AS t(elem, ord) + WHERE elem ->> 's' = selector + ORDER BY ord +$$; + +COMMENT ON FUNCTION {{prefix}}.json_tree_path_query(jsonb, text) IS + 'eql-inline-critical: raw-jsonb path query helper; must stay inlinable (unpinned search_path)'; + +--! @brief The first entry whose path selector is `selector`, or NULL. +--! @param val jsonb The document. +--! @param selector text The path selector, hex. +--! @return public.{{prefix}}_json_tree_entry The entry, or NULL when the document +--! has no node at that path. +CREATE FUNCTION {{prefix}}.json_tree_path_query_first(val jsonb, selector text) + RETURNS public.{{prefix}}_json_tree_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}_internal.json_tree_graft(val, elem, (ord - 1)::integer) + FROM jsonb_array_elements(val -> 'sv') WITH ORDINALITY AS t(elem, ord) + WHERE elem ->> 's' = selector + ORDER BY ord + LIMIT 1 +$$; + +COMMENT ON FUNCTION {{prefix}}.json_tree_path_query_first(jsonb, text) IS + 'eql-inline-critical: raw-jsonb path first helper; must stay inlinable (unpinned search_path)'; + +--! @brief Whether the document has a node at the path `selector` names. +--! @param val jsonb The document. +--! @param selector text The path selector, hex. +--! @return boolean True when some entry carries the selector. +CREATE FUNCTION {{prefix}}.json_tree_path_exists(val jsonb, selector text) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'sv') AS elem + WHERE elem ->> 's' = selector + ) +$$; + +COMMENT ON FUNCTION {{prefix}}.json_tree_path_exists(jsonb, text) IS + 'eql-inline-critical: raw-jsonb path exists helper; must stay inlinable (unpinned search_path)'; + +------------------------------------------------------------------------------ +-- Containment: the element-nested projection +------------------------------------------------------------------------------ + +--! @brief One group of a stored document's projection, from its group tags. +--! @internal +--! +--! An entry belongs to the group of its own tag `n` when it is an array +--! element that is a container, else to the group of its enclosing element's +--! tag `g`, else to the top level. A group's `el` holds one nested group per +--! container element whose `g` is the group's tag. The needle is projected +--! the same way by the client, from its structure, so `jsonb @>` over the two +--! is PostgreSQL's own containment rule: each needle element has to fit +--! inside one stored element. +--! +--! @param sv jsonb The document's `sv` array. +--! @param parent text The group's tag, hex, or NULL for the top level. +--! @return jsonb `{"sv": [{"s", "vs"}, …], "el": [group, …]}`. +--! @note plpgsql: recursive. IMMUTABLE and deterministic (document order), so +--! `{{prefix}}.json_tree_query` over it serves an expression index. +CREATE FUNCTION {{prefix}}_internal.json_tree_group(sv jsonb, parent text) + RETURNS jsonb + LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE +AS $$ +DECLARE + pairs jsonb; + groups jsonb; +BEGIN + SELECT COALESCE(jsonb_agg(jsonb_build_object('s', elem -> 's', 'vs', elem -> 'vs') ORDER BY ord), '[]'::jsonb) + INTO pairs + FROM jsonb_array_elements(sv) WITH ORDINALITY AS t(elem, ord) + WHERE COALESCE(elem ->> 'n', elem ->> 'g') IS NOT DISTINCT FROM parent; + SELECT COALESCE(jsonb_agg({{prefix}}_internal.json_tree_group(sv, elem ->> 'n') ORDER BY ord), '[]'::jsonb) + INTO groups + FROM jsonb_array_elements(sv) WITH ORDINALITY AS t(elem, ord) + WHERE elem ? 'n' AND (elem ->> 'g') IS NOT DISTINCT FROM parent; + RETURN jsonb_build_object('sv', pairs, 'el', groups); +END; +$$; + +--! @brief Project a stored document into its containment shape: the needle +--! shape `{{prefix}}.query_json_tree` holds, from the document's group +--! tags. +--! +--! This is the GIN index expression for containment: +--! `CREATE INDEX … USING gin (({{prefix}}.json_tree_query(col)::jsonb) jsonb_path_ops);` +--! The typed `@>` inlines to native `jsonb @>` over the same expression, so +--! the index engages; GIN is lossy and the executor rechecks the real +--! operator on the heap tuple, which is the exact containment. +--! +--! @param e public.{{prefix}}_json_tree The stored document. +--! @return {{prefix}}.query_json_tree Its projection. +CREATE FUNCTION {{prefix}}.json_tree_query(e public.{{prefix}}_json_tree) + RETURNS {{prefix}}.query_json_tree + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}_internal.json_tree_group(e::jsonb -> 'sv', NULL)::{{prefix}}.query_json_tree +$$; + +--! @brief Does the stored document `a` contain the needle `b`? +--! +--! Native `jsonb @>` over the projections, plus the needle's `alt` projection +--! when it carries one (a root scalar against a stored root array). Both +--! probes are over `{{prefix}}.json_tree_query(a)::jsonb`, so a GIN index on +--! that expression serves them as one bitmap OR. +--! +--! @param a public.{{prefix}}_json_tree The stored document. +--! @param b {{prefix}}.query_json_tree The needle. +--! @return boolean True when `a` contains `b` as `jsonb @>` would contain the +--! plaintexts; NULL when either is NULL. +--! @note Deliberately not STRICT. The planner inlines a STRICT SQL function +--! only when it can prove the body is NULL for a NULL argument, and +--! `x OR y` is not provably NULL. Un-inlined, the call is an opaque +--! filter and the GIN index never engages. The body returns NULL for +--! a NULL argument on its own: a NULL side of `@>` is NULL, and NULL +--! OR NULL is NULL. +CREATE FUNCTION {{prefix}}.json_tree_contains(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_query(a)::jsonb @> (b::jsonb - 'alt') + OR ((b::jsonb ? 'alt') AND {{prefix}}.json_tree_query(a)::jsonb @> (b::jsonb -> 'alt')) +$$; + +--! @brief Is the stored document `a` contained by the needle `b`? +--! +--! The projections the other way round. A stored document carries no `alt`, +--! so a stored root scalar inside a needle root array is the one case this +--! answers false where `jsonb <@` answers true. +--! +--! @param a public.{{prefix}}_json_tree The stored document. +--! @param b {{prefix}}.query_json_tree The needle. +--! @return boolean True when `b` contains `a`. +CREATE FUNCTION {{prefix}}.json_tree_contained_by(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT (b::jsonb - 'alt') @> {{prefix}}.json_tree_query(a)::jsonb +$$; + +--! @brief Does the stored document `a` contain the stored document `b`? +--! @param a public.{{prefix}}_json_tree The container. +--! @param b public.{{prefix}}_json_tree The contained document. +--! @return boolean True when `a`'s projection contains `b`'s. +CREATE FUNCTION {{prefix}}.json_tree_document_contains(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_query(a)::jsonb @> {{prefix}}.json_tree_query(b)::jsonb +$$; + +------------------------------------------------------------------------------ +-- The token chain: children and subtrees by token +------------------------------------------------------------------------------ + +--! @brief The children of the node whose token is `token`, as rows. +--! +--! For every ordinal under the document's fan-out, the parent tag the child +--! at that ordinal would carry; the entries carrying one are the children, +--! and each row's token is unwrapped on the way, though not returned. What +--! `jsonb_array_elements` is for an encrypted array: one row per element, in +--! ordinal order, each opening on its own. +--! +--! @param e public.{{prefix}}_json_tree The stored document. +--! @param token text The parent's token, hex: derived by the client from the +--! path alone, and never stored. +--! @return SETOF public.{{prefix}}_json_tree_entry The children, each with `d = 1`. +CREATE FUNCTION {{prefix}}.json_tree_children(e public.{{prefix}}_json_tree, token text) + RETURNS SETOF public.{{prefix}}_json_tree_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT ({{prefix}}_internal.json_tree_graft(e::jsonb, c.elem, (c.ord - 1)::integer)::jsonb || '{"d": 1}'::jsonb)::public.{{prefix}}_json_tree_entry + FROM generate_series(0, {{prefix}}_internal.json_tree_fanout(e::jsonb) - 1) AS o + JOIN LATERAL ( + SELECT elem, ord + FROM jsonb_array_elements(e::jsonb -> 'sv') WITH ORDINALITY AS t(elem, ord) + WHERE elem ->> 'p' = {{prefix}}_internal.json_tree_parent_tag( + {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h'), decode(token, 'hex'), o) + ) AS c ON true + ORDER BY o +$$; + +--! @brief How many children the node whose token is `token` has: the length +--! of an encrypted array, or the member count of an object. +--! @param e public.{{prefix}}_json_tree The stored document. +--! @param token text The node's token, hex. +--! @return integer The count; `0` for a node with no children, a token that +--! names no node included. +CREATE FUNCTION {{prefix}}.json_tree_child_count(e public.{{prefix}}_json_tree, token text) + RETURNS integer + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT count(*)::integer FROM {{prefix}}.json_tree_children(e, token) +$$; + +--! @brief Every entry under the node whose token is `token`, as rows: the +--! recursive walk of the chain, in core SQL. +--! +--! Starting from the token, each level finds its children by parent tag, +--! unwraps their tokens, and recurses. A token that names no node yields no +--! row, and nothing in a row is linkable to anything without the token. The +--! node itself is not returned: the walk addresses nodes by their parent, +--! and the client reads the node's kind from its children's labels, or +--! fetches the node by its selector. +--! The walk descends at most 127 levels, the deepest a sealed document +--! nests, so a stored document whose tags do not form a tree still ends it. +--! +--! @param e public.{{prefix}}_json_tree The stored document. +--! @param token text The subtree's token, hex. +--! @return SETOF public.{{prefix}}_json_tree_entry The descendants, each with its +--! depth `d` below the node, in depth-first order. +CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_tree, token text) + RETURNS SETOF public.{{prefix}}_json_tree_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + WITH RECURSIVE + doc AS ( + SELECT e::jsonb AS val, + {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h') AS iv, + {{prefix}}_internal.json_tree_fanout(e::jsonb) AS fanout + ), + walk(depth, t, j, entry, path) AS ( + SELECT 0, decode(token, 'hex'), NULL::integer, NULL::jsonb, ARRAY[]::integer[] + UNION ALL + SELECT w.depth + 1, + {{prefix}}_internal.json_tree_unwrap(doc.iv, w.t, c.elem), + (c.ord - 1)::integer, + c.elem, + w.path || (c.ord - 1)::integer + FROM walk AS w + CROSS JOIN doc + JOIN LATERAL ( + SELECT elem, ord + FROM generate_series(0, doc.fanout - 1) AS o + JOIN LATERAL ( + SELECT elem, ord + FROM jsonb_array_elements(doc.val -> 'sv') WITH ORDINALITY AS t(elem, ord) + WHERE elem ->> 'p' = {{prefix}}_internal.json_tree_parent_tag(doc.iv, w.t, o) + ) AS hit ON true + ) AS c ON true + -- No sealed document nests deeper than 127 levels. A stored one + -- whose tags form a cycle or a longer chain, which whoever holds a + -- token can write, stops here instead of recursing without end. + WHERE w.depth < 127 + ) + SELECT ({{prefix}}_internal.json_tree_graft(doc.val, w.entry, w.j)::jsonb || jsonb_build_object('d', w.depth))::public.{{prefix}}_json_tree_entry + FROM walk AS w CROSS JOIN doc + WHERE w.entry IS NOT NULL + ORDER BY w.path +$$; + +--! @brief The subtrees under the node whose token is `token` whose path +--! continues through a node with selector `selector`: how a path +--! with a wildcard below its last positional step is walked. +--! +--! The walk starts at the token and emits an entry once it has passed +--! through, or reached, an entry carrying the selector: for +--! `$.orders[*].items` it starts at `$.orders` and returns every element's +--! `items` subtree, each complete. +--! +--! @param e public.{{prefix}}_json_tree The stored document. +--! @param token text The anchor's token, hex. +--! @param selector text The whole path's selector, hex. +--! @return SETOF public.{{prefix}}_json_tree_entry The matching subtrees' entries, +--! each with its depth `d` below the anchor, in depth-first order. +CREATE FUNCTION {{prefix}}.json_tree_subtree_at(e public.{{prefix}}_json_tree, token text, selector text) + RETURNS SETOF public.{{prefix}}_json_tree_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + WITH RECURSIVE + doc AS ( + SELECT e::jsonb AS val, + {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h') AS iv, + {{prefix}}_internal.json_tree_fanout(e::jsonb) AS fanout + ), + walk(depth, t, j, entry, path, hit) AS ( + SELECT 0, decode(token, 'hex'), NULL::integer, NULL::jsonb, ARRAY[]::integer[], false + UNION ALL + SELECT w.depth + 1, + {{prefix}}_internal.json_tree_unwrap(doc.iv, w.t, c.elem), + (c.ord - 1)::integer, + c.elem, + w.path || (c.ord - 1)::integer, + w.hit OR c.elem ->> 's' = selector + FROM walk AS w + CROSS JOIN doc + JOIN LATERAL ( + SELECT elem, ord + FROM generate_series(0, doc.fanout - 1) AS o + JOIN LATERAL ( + SELECT elem, ord + FROM jsonb_array_elements(doc.val -> 'sv') WITH ORDINALITY AS t(elem, ord) + WHERE elem ->> 'p' = {{prefix}}_internal.json_tree_parent_tag(doc.iv, w.t, o) + ) AS hit ON true + ) AS c ON true + -- No sealed document nests deeper than 127 levels. A stored one + -- whose tags form a cycle or a longer chain, which whoever holds a + -- token can write, stops here instead of recursing without end. + WHERE w.depth < 127 + ) + SELECT ({{prefix}}_internal.json_tree_graft(doc.val, w.entry, w.j)::jsonb || jsonb_build_object('d', w.depth))::public.{{prefix}}_json_tree_entry + FROM walk AS w CROSS JOIN doc + WHERE w.entry IS NOT NULL AND w.hit + ORDER BY w.path +$$; + +------------------------------------------------------------------------------ +-- Ordering at a path +------------------------------------------------------------------------------ + +--! @brief The path selector of an entry. +--! @param entry public.{{prefix}}_json_tree_entry The entry. +--! @return text Its `s`, hex. +CREATE FUNCTION {{prefix}}.selector(entry public.{{prefix}}_json_tree_entry) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT entry ->> 's' +$$; + +--! @brief The leaf ordering term of an entry, for ordered comparison. +--! +--! The CLLW OPE term the index derived for a number or string leaf, keyed by +--! the column, hex-decoded to the bytea-backed `{{prefix}}_internal.ope_cllw` +--! domain: native byte order is plaintext order, so a functional btree on +--! `{{prefix}}.ord_term(col -> 'selector')` engages with the DEFAULT opclass. NULL +--! for an entry with no term (a container, a boolean, a null), which btree's +--! NULL handling drops from range queries. +--! +--! Order rows by this, `ORDER BY {{prefix}}.ord_term(col -> 'sel')`, never by +--! the entry itself: `ORDER BY col -> 'sel'` uses jsonb's default ordering of +--! the entry object, which sorts by ciphertext, and an operator class that +--! would redirect it needs a superuser to install. +--! +--! @param entry public.{{prefix}}_json_tree_entry The entry. +--! @return {{prefix}}_internal.ope_cllw The term, or NULL. +CREATE FUNCTION {{prefix}}.ord_term(entry public.{{prefix}}_json_tree_entry) + RETURNS {{prefix}}_internal.ope_cllw + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}_internal.ope_cllw(entry::jsonb) +$$; + +--! @brief The leaf ordering term of an ordering operand. +--! @param q {{prefix}}.query_json_tree_leaf The operand. +--! @return {{prefix}}_internal.ope_cllw Its `op`, hex-decoded. +CREATE FUNCTION {{prefix}}.ord_term(q {{prefix}}.query_json_tree_leaf) + RETURNS {{prefix}}_internal.ope_cllw + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}_internal.ope_cllw(q::jsonb) +$$; + +--! @brief The path selector of an ordering operand: what `->` takes to reach +--! the leaf the operand compares against. +--! @param q {{prefix}}.query_json_tree_leaf The operand. +--! @return text Its `s`, hex. +CREATE FUNCTION {{prefix}}.selector(q {{prefix}}.query_json_tree_leaf) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT q ->> 's' +$$; diff --git a/packages/eql/src/v3/json_tree/operators.sql b/packages/eql/src/v3/json_tree/operators.sql new file mode 100644 index 000000000..e4c75d78c --- /dev/null +++ b/packages/eql/src/v3/json_tree/operators.sql @@ -0,0 +1,329 @@ +-- ONLY EQL VERSION: 4 +-- REQUIRE: src/v3/schema.sql +-- REQUIRE: src/v3/json_tree/types.sql +-- REQUIRE: src/v3/json_tree/functions.sql +-- REQUIRE: src/v3/sem/ope_cllw/types.sql + +--! @file v3/json_tree/operators.sql +--! @brief Operators on public.{{prefix}}_json_tree and public.{{prefix}}_json_tree_entry. + +------------------------------------------------------------------------------ +-- -> and ->> accessors (by selector) +------------------------------------------------------------------------------ + +--! @brief -> operator with a text selector: the first entry at the path. +--! +--! Returns the entry whose `s` equals @p selector, with the document's +--! `h`, `i`, `v` and the entry's ordinal `j` grafted on, so it opens on its +--! own. Inlinable: `{{prefix}}.ord_term(col -> 'sel')` matches a functional +--! btree on that expression. +--! +--! @warning The selector operand MUST carry a known type: a text-typed +--! parameter or an explicit cast (`col -> 'sel'::text`). A bare untyped +--! literal resolves to the NATIVE `jsonb -> text` operator and silently +--! returns native jsonb semantics, as it does on every EQL jsonb domain. +--! +--! @param e public.{{prefix}}_json_tree The stored document. +--! @param selector text The path selector, hex. +--! @return public.{{prefix}}_json_tree_entry The entry, or NULL. +CREATE FUNCTION {{prefix}}."->"(e public.{{prefix}}_json_tree, selector text) + RETURNS public.{{prefix}}_json_tree_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_path_query_first(e::jsonb, selector) +$$; + +CREATE OPERATOR ->( + FUNCTION={{prefix}}."->", + LEFTARG=public.{{prefix}}_json_tree, + RIGHTARG=text +); + +--! @brief ->> operator with a text selector: the first entry at the path, as +--! JSON text. It does not decrypt. +--! @param e public.{{prefix}}_json_tree The stored document. +--! @param selector text The path selector, hex. +--! @return text The entry as text, or NULL. +CREATE FUNCTION {{prefix}}."->>"(e public.{{prefix}}_json_tree, selector text) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}."->"(e, selector)::jsonb::text +$$; + +CREATE OPERATOR ->> ( + FUNCTION={{prefix}}."->>", + LEFTARG=public.{{prefix}}_json_tree, + RIGHTARG=text +); + +------------------------------------------------------------------------------ +-- @> and <@ containment +------------------------------------------------------------------------------ + +--! @brief @> contains operator with a query_json_tree needle. +--! +--! Inlines to native `jsonb @>` over `{{prefix}}.json_tree_query(a)::jsonb`, so +--! a functional GIN index on that expression engages. Not STRICT, for the +--! reason `{{prefix}}.json_tree_contains` gives: a STRICT wrapper around a +--! non-STRICT body is not inlined. +--! +--! @param a public.{{prefix}}_json_tree The container. +--! @param b {{prefix}}.query_json_tree The needle. +--! @return boolean True when `a` contains `b`; NULL when either is NULL. +--! @see {{prefix}}.json_tree_contains +CREATE FUNCTION {{prefix}}."@>"(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_contains(a, b) +$$; + +CREATE OPERATOR @>( + FUNCTION={{prefix}}."@>", + LEFTARG=public.{{prefix}}_json_tree, + RIGHTARG={{prefix}}.query_json_tree +); + +--! @brief @> contains operator (document, document). +--! @param a public.{{prefix}}_json_tree The container. +--! @param b public.{{prefix}}_json_tree The contained document. +--! @return boolean True when `a` contains `b`. +CREATE FUNCTION {{prefix}}."@>"(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_document_contains(a, b) +$$; + +CREATE OPERATOR @>( + FUNCTION={{prefix}}."@>", + LEFTARG=public.{{prefix}}_json_tree, + RIGHTARG=public.{{prefix}}_json_tree +); + +--! @brief <@ contained-by operator with a query_json_tree container. +--! @param a public.{{prefix}}_json_tree The contained document. +--! @param b {{prefix}}.query_json_tree The needle that must contain it. +--! @return boolean True when `b` contains `a`. +--! @see {{prefix}}.json_tree_contained_by +CREATE FUNCTION {{prefix}}."<@"(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_contained_by(a, b) +$$; + +CREATE OPERATOR <@( + FUNCTION={{prefix}}."<@", + LEFTARG=public.{{prefix}}_json_tree, + RIGHTARG={{prefix}}.query_json_tree +); + +--! @brief <@ contained-by operator with a query_json_tree LHS: `b @> a`. +--! Not STRICT, as `@>` is not. +--! @param a {{prefix}}.query_json_tree The needle. +--! @param b public.{{prefix}}_json_tree The container. +--! @return boolean True when `b` contains `a`; NULL when either is NULL. +CREATE FUNCTION {{prefix}}."<@"(a {{prefix}}.query_json_tree, b public.{{prefix}}_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_contains(b, a) +$$; + +CREATE OPERATOR <@( + FUNCTION={{prefix}}."<@", + LEFTARG={{prefix}}.query_json_tree, + RIGHTARG=public.{{prefix}}_json_tree +); + +--! @brief <@ contained-by operator (document, document). +--! @param a public.{{prefix}}_json_tree The contained document. +--! @param b public.{{prefix}}_json_tree The container. +--! @return boolean True when `b` contains `a`. +CREATE FUNCTION {{prefix}}."<@"(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_document_contains(b, a) +$$; + +CREATE OPERATOR <@( + FUNCTION={{prefix}}."<@", + LEFTARG=public.{{prefix}}_json_tree, + RIGHTARG=public.{{prefix}}_json_tree +); + +------------------------------------------------------------------------------ +-- Ordering at a path: entry against an ordering operand +------------------------------------------------------------------------------ + +--! @brief Less-than on an extracted entry against an ordering operand, via +--! the CLLW OPE term (native bytea order). +--! @param a public.{{prefix}}_json_tree_entry The entry, `col -> 'sel'`. +--! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @return boolean True when the entry's leaf is below the operand's. +CREATE FUNCTION {{prefix}}.lt(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) < {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR < ( + FUNCTION = {{prefix}}.lt, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = {{prefix}}.query_json_tree_leaf, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + +--! @brief Less-than-or-equal on an extracted entry against an ordering operand. +--! @param a public.{{prefix}}_json_tree_entry The entry. +--! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @return boolean True when the entry's leaf is at or below the operand's. +CREATE FUNCTION {{prefix}}.lte(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) <= {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR <= ( + FUNCTION = {{prefix}}.lte, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = {{prefix}}.query_json_tree_leaf, + NEGATOR = >, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel +); + +--! @brief Greater-than on an extracted entry against an ordering operand. +--! @param a public.{{prefix}}_json_tree_entry The entry. +--! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @return boolean True when the entry's leaf is above the operand's. +CREATE FUNCTION {{prefix}}.gt(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) > {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR > ( + FUNCTION = {{prefix}}.gt, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = {{prefix}}.query_json_tree_leaf, + NEGATOR = <=, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel +); + +--! @brief Greater-than-or-equal on an extracted entry against an ordering operand. +--! @param a public.{{prefix}}_json_tree_entry The entry. +--! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @return boolean True when the entry's leaf is at or above the operand's. +CREATE FUNCTION {{prefix}}.gte(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) >= {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR >= ( + FUNCTION = {{prefix}}.gte, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = {{prefix}}.query_json_tree_leaf, + NEGATOR = <, + RESTRICT = scalargesel, + JOIN = scalargejoinsel +); + +------------------------------------------------------------------------------ +-- Ordering between two extracted entries +------------------------------------------------------------------------------ + +--! @brief Less-than between two extracted entries via their OPE terms. +--! @param a public.{{prefix}}_json_tree_entry Left operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @return boolean True when `a`'s leaf is below `b`'s. +CREATE FUNCTION {{prefix}}.lt(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) < {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR < ( + FUNCTION = {{prefix}}.lt, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = public.{{prefix}}_json_tree_entry, + COMMUTATOR = >, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + +--! @brief Less-than-or-equal between two extracted entries. +--! @param a public.{{prefix}}_json_tree_entry Left operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @return boolean True when `a`'s leaf is at or below `b`'s. +CREATE FUNCTION {{prefix}}.lte(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) <= {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR <= ( + FUNCTION = {{prefix}}.lte, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = public.{{prefix}}_json_tree_entry, + COMMUTATOR = >=, + NEGATOR = >, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel +); + +--! @brief Greater-than between two extracted entries. +--! @param a public.{{prefix}}_json_tree_entry Left operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @return boolean True when `a`'s leaf is above `b`'s. +CREATE FUNCTION {{prefix}}.gt(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) > {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR > ( + FUNCTION = {{prefix}}.gt, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = public.{{prefix}}_json_tree_entry, + COMMUTATOR = <, + NEGATOR = <=, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel +); + +--! @brief Greater-than-or-equal between two extracted entries. +--! @param a public.{{prefix}}_json_tree_entry Left operand. +--! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @return boolean True when `a`'s leaf is at or above `b`'s. +CREATE FUNCTION {{prefix}}.gte(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.ord_term(a) >= {{prefix}}.ord_term(b) +$$; + +CREATE OPERATOR >= ( + FUNCTION = {{prefix}}.gte, + LEFTARG = public.{{prefix}}_json_tree_entry, + RIGHTARG = public.{{prefix}}_json_tree_entry, + COMMUTATOR = <=, + NEGATOR = <, + RESTRICT = scalargesel, + JOIN = scalargejoinsel +); diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql new file mode 100644 index 000000000..abedea647 --- /dev/null +++ b/packages/eql/src/v3/json_tree/types.sql @@ -0,0 +1,267 @@ +-- ONLY EQL VERSION: 4 +-- REQUIRE: src/v3/schema.sql + +--! @file v3/json_tree/types.sql +--! @brief Domain types for the {{prefix}} JSON tree surface: an encrypted JSON +--! document stored as one entry per node, with node-local ciphertexts, +--! a token chain and exact containment. +--! +--! This is a new domain family beside `public.{{prefix}}_json`, for documents +--! Stack Encrypt's JSON index writes. It shares no wire shape and no SQL with +--! the SteVec surface. Four jsonb-backed domains (none over another domain, +--! so the native-jsonb firewall in blockers.sql attaches): +--! - public.{{prefix}}_json_tree — storage: an EQL envelope `{v, i, h, sv}`. +--! - public.{{prefix}}_json_tree_entry — one `sv` element (returned by `->` +--! and the subtree walk). +--! - {{prefix}}.query_json_tree — a containment needle: the element-nested +--! projection of the needle's selectors. +--! - {{prefix}}.query_json_tree_leaf — an ordering operand at a path: a +--! selector and a leaf ordering term. +--! +--! The design is `docs/plans/2026-10-17-json-tree-index.md` +--! at the repository root. The document's per-entry fields: +--! `s` path selector (16 bytes, hex): every element of an array shares its +--! parent's, so a path reads `$.a[*].x`. +--! `vs` value selector (hex): the node's kind and, for a scalar, its value, +--! bound to `s`. Containment is a set comparison over `{s, vs}` pairs. +--! `c` the node's own ciphertext (base64): its label, its kind and, for a +--! scalar, its value. The root's is the whole document when the root is +--! sealed (the default). +--! `o` the node's ordinal under its parent, `p` the parent tag +--! `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))` and `w` the node's own token +--! wrapped under its parent's: present below the chain's heads. +--! `n` the group tag of an array element that is a container, +--! `g` the group tag of the nearest enclosing such element. +--! `op` the leaf ordering term (CLLW OPE, hex) on number and string leaves. +--! The header `h` (base64) carries the document's data key material once; the +--! SQL reads only the ZeroKMS `iv` at byte offset 22 of it, the per-document +--! salt of the chain tags, and otherwise carries it opaquely. + +--! @brief Validate one JSON tree entry. +--! @internal +--! @param val jsonb Candidate entry. +--! @return boolean True when `val` is an object with string `s`, `vs` and `c`, +--! an optional numeric `o`, optional string `p`, `w`, `n`, `g` and +--! `op`, and the metadata the extractors graft: string `h`, numeric +--! `j` (the entry's ordinal in the document) and `d` (its depth below +--! a walk's start), and the envelope's `v` and `i`. No other key. +CREATE OR REPLACE FUNCTION public.{{prefix}}_is_valid_json_tree_entry_payload(val jsonb) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT COALESCE( + jsonb_typeof(val) = 'object' + AND jsonb_typeof(val -> 's') = 'string' + AND jsonb_typeof(val -> 'vs') = 'string' + AND jsonb_typeof(val -> 'c') = 'string' + AND (NOT (val ? 'o') OR jsonb_typeof(val -> 'o') = 'number') + AND (NOT (val ? 'p') OR jsonb_typeof(val -> 'p') = 'string') + AND (NOT (val ? 'w') OR jsonb_typeof(val -> 'w') = 'string') + AND (NOT (val ? 'n') OR jsonb_typeof(val -> 'n') = 'string') + AND (NOT (val ? 'g') OR jsonb_typeof(val -> 'g') = 'string') + AND (NOT (val ? 'op') OR jsonb_typeof(val -> 'op') = 'string') + AND (NOT (val ? 'h') OR jsonb_typeof(val -> 'h') = 'string') + AND (NOT (val ? 'j') OR jsonb_typeof(val -> 'j') = 'number') + AND (NOT (val ? 'd') OR jsonb_typeof(val -> 'd') = 'number') + AND val - ARRAY['s', 'vs', 'c', 'o', 'p', 'w', 'n', 'g', 'op', 'h', 'j', 'd', 'v', 'i']::text[] = '{}'::jsonb, + false + ) +$$; + +--! @brief Validate a JSON tree document. +--! @internal +--! @param val jsonb Candidate document. +--! @return boolean True when `val` is an envelope with `v = {{eql_version}}`, `i`, a +--! string header `h` and a non-empty `sv` array of valid entries, and +--! no other key. A document always has a root, so `sv` is never empty. +CREATE OR REPLACE FUNCTION public.{{prefix}}_is_valid_json_tree_document_payload(val jsonb) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT COALESCE( + jsonb_typeof(val) = 'object' + AND val ? 'v' + AND val ->> 'v' = '{{eql_version}}' + AND val ? 'i' + AND jsonb_typeof(val -> 'h') = 'string' + AND jsonb_typeof(val -> 'sv') = 'array' + AND jsonb_array_length(val -> 'sv') >= 1 + AND val - ARRAY['v', 'i', 'h', 'sv']::text[] = '{}'::jsonb + AND NOT EXISTS ( + SELECT 1 + FROM jsonb_array_elements(val -> 'sv') AS elem + WHERE NOT public.{{prefix}}_is_valid_json_tree_entry_payload(elem) + ), + false + ) +$$; + +--! @brief Validate one group of a containment needle, recursively. +--! @internal +--! +--! A group is `{"sv": [{"s": "…", "vs": "…"}, …], "el": [group, …]}`: the +--! selector pairs that sit directly in it, and one nested group per array +--! element that is a container. The same shape `{{prefix}}.json_tree_query` +--! projects a stored document into, so native `jsonb @>` over the two is +--! exact containment. +--! +--! @param val jsonb Candidate group. +--! @return boolean True when `val` is a group. +--! @note plpgsql: the recursion and the subqueries over the arrays cannot be +--! written as a CHECK expression, and a plpgsql body caches its plan +--! across the per-needle casts on the query hot path. +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_query_group(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + RETURN COALESCE( + jsonb_typeof(val) = 'object' + AND jsonb_typeof(val -> 'sv') = 'array' + AND jsonb_typeof(val -> 'el') = 'array' + AND val - ARRAY['sv', 'el']::text[] = '{}'::jsonb + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'sv') AS pair + WHERE NOT COALESCE( + jsonb_typeof(pair) = 'object' + AND jsonb_typeof(pair -> 's') = 'string' + AND jsonb_typeof(pair -> 'vs') = 'string' + AND pair - ARRAY['s', 'vs']::text[] = '{}'::jsonb, + false) + ) + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'el') AS grp + WHERE NOT {{prefix}}_internal.json_tree_is_valid_query_group(grp) + ), + false + ); +END; +$$; + +--! @brief Validate a containment needle. +--! @internal +--! @param val jsonb Candidate needle. +--! @return boolean True when `val` is a group (`sv`, `el`) with an optional +--! `alt` group beside it: the alternative projection a needle whose +--! root is a scalar carries, so that a stored root array holding the +--! scalar matches as `jsonb @>` says it does. +CREATE OR REPLACE FUNCTION public.{{prefix}}_is_valid_json_tree_query_payload(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + RETURN COALESCE( + jsonb_typeof(val) = 'object' + AND {{prefix}}_internal.json_tree_is_valid_query_group(val - 'alt') + AND (NOT (val ? 'alt') OR {{prefix}}_internal.json_tree_is_valid_query_group(val -> 'alt')), + false + ); +END; +$$; + +--! @brief Storage domain for an encrypted JSON tree column. +--! +--! CHECK: the EQL envelope (`v = {{eql_version}}`, `i`, `h`) around a non-empty `sv` +--! array of valid entries. The firewall in blockers.sql attaches to this +--! domain to stop native jsonb operators from reaching a column value. +--! +--! @note Constructing from inline JSON uses the standard DOMAIN cast: +--! `'{"v":{{eql_version}},"i":{…},"h":"…","sv":[…]}'::public.{{prefix}}_json_tree`. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = '{{prefix}}_json_tree' AND typnamespace = 'public'::regnamespace + ) THEN + CREATE DOMAIN public.{{prefix}}_json_tree AS jsonb + CHECK ( + public.{{prefix}}_is_valid_json_tree_document_payload(VALUE) + ); + END IF; + + COMMENT ON DOMAIN public.{{prefix}}_json_tree IS 'EQL encrypted JSON tree document (containment, token chain)'; +END +$$; + +--! @brief Domain type for one entry of a JSON tree document. +--! +--! What `->`, the path functions and the subtree walk return: the entry with +--! the document's `h`, `v` and `i` and its own ordinal `j` grafted on, so it +--! opens on its own. A walk also grafts `d`, the entry's depth below the +--! walk's start. +--! +--! @note The CHECK calls the validator rather than inlining it: an entry is +--! constructed by the extractors, not cast on the query hot path. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = '{{prefix}}_json_tree_entry' AND typnamespace = 'public'::regnamespace + ) THEN + CREATE DOMAIN public.{{prefix}}_json_tree_entry AS jsonb + CHECK ( + VALUE IS NULL OR public.{{prefix}}_is_valid_json_tree_entry_payload(VALUE) + ); + END IF; + + COMMENT ON DOMAIN public.{{prefix}}_json_tree_entry IS 'EQL encrypted JSON tree entry (one node)'; +END +$$; + +--! @brief Domain type for a JSON tree containment needle. +--! +--! The element-nested projection of the needle's selectors, as Stack +--! Encrypt's `JsonQuery` serializes it: `{"sv": [{"s", "vs"}, …], "el": +--! [group, …], "alt": group?}`. Typing the needle this way stops raw jsonb +--! from casting and matching every row via bare `jsonb @>`. +--! +--! @note Construct from inline JSON via the DOMAIN cast: +--! `'{"sv":[…],"el":[]}'::{{prefix}}.query_json_tree`. +--! @see {{prefix}}.json_tree_query +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = 'query_json_tree' AND typnamespace = '{{prefix}}'::regnamespace + ) THEN + CREATE DOMAIN {{prefix}}.query_json_tree AS jsonb + CHECK ( + public.{{prefix}}_is_valid_json_tree_query_payload(VALUE) + ); + END IF; + + COMMENT ON DOMAIN {{prefix}}.query_json_tree IS 'EQL JSON tree query operand (containment)'; +END +$$; + +--! @brief Domain type for an ordering operand at a JSON path. +--! +--! `{"s": "…", "op": "…"}`: the path's selector, which `->` takes, and the +--! leaf ordering term the stored leaf's `op` compares against, as Stack +--! Encrypt's `JsonLeafQuery` serializes it. One operand serves `<`, `<=`, +--! `>` and `>=` on an extracted entry; a range at a path is two predicates. +--! +--! @note Construct from inline JSON via the DOMAIN cast: +--! `'{"s":"…","op":"…"}'::{{prefix}}.query_json_tree_leaf`. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = 'query_json_tree_leaf' AND typnamespace = '{{prefix}}'::regnamespace + ) THEN + CREATE DOMAIN {{prefix}}.query_json_tree_leaf AS jsonb + CHECK ( + VALUE IS NULL + OR COALESCE( + jsonb_typeof(VALUE) = 'object' + AND jsonb_typeof(VALUE -> 's') = 'string' + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND VALUE - ARRAY['s', 'op']::text[] = '{}'::jsonb, + false + ) + ); + END IF; + + COMMENT ON DOMAIN {{prefix}}.query_json_tree_leaf IS 'EQL JSON tree query operand (ordering at a path)'; +END +$$; diff --git a/packages/eql/src/v3/lint/lints.sql b/packages/eql/src/v3/lint/lints.sql index 1e87deade..54e798459 100644 --- a/packages/eql/src/v3/lint/lints.sql +++ b/packages/eql/src/v3/lint/lints.sql @@ -19,7 +19,7 @@ --! (or `pg_get_functiondef`); tracked as a follow-up. --! --! Operators on `{{prefix}}` types (the jsonb-backed encrypted-domain families and ---! the SEM index-term type `{{prefix}}_internal.ore_block_256`) whose +--! the SEM index-term type `{{prefix}}_internal.{{ore_block}}`) whose --! implementation functions fail any of these rules silently fall back to seq --! scan when the documented functional indexes (`{{prefix}}.eq_term(col)`, --! `{{prefix}}.ord_term(col)`) are in place. This lint surfaces every such case. @@ -58,7 +58,7 @@ --! functional index on the extractor instead. --! `schema_placement` — a naked composite or enum TYPE lives in the --! public `{{prefix}}` schema. Internal index-term ---! types (e.g. `ore_block_256_term`) belong in +--! types (e.g. `{{ore_block}}_term`) belong in --! `{{prefix}}_internal`; a composite/enum in --! `{{prefix}}` clutters the Supabase Table Builder --! type picker, which the schema split exists to diff --git a/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql index c2856dba4..aedbcafc2 100644 --- a/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_bigint_ord_ore. --! @param a public.{{prefix}}_bigint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_bigint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_bigint_ord_ore. --! @param a public.{{prefix}}_bigint_ord_ore diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql index c1dec7671..8aa94e2f8 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_bigint_ord_ore. --! @param a {{prefix}}.query_bigint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_bigint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_bigint_ord_ore. --! @param a public.{{prefix}}_bigint_ord_ore diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql index 0d9edb9c8..418810bc8 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_bigint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_bigint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_bigint_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_bigint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_bigint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_bigint_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_bigint_ord`, `public.{{prefix}}_bigint_ord_ope`) diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql index 37c422c54..476d4f9c4 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql b/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql index 676c12d33..6c7da4429 100644 --- a/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_date_ord_ore. --! @param a public.{{prefix}}_date_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_date_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_date_ord_ore. --! @param a public.{{prefix}}_date_ord_ore diff --git a/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql b/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql index c06aebf87..5150c1f40 100644 --- a/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_date_ord_ore. --! @param a {{prefix}}.query_date_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_date_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_date_ord_ore. --! @param a public.{{prefix}}_date_ord_ore diff --git a/packages/eql/src/v3/scalars/date/query_date_range_functions.sql b/packages/eql/src/v3/scalars/date/query_date_range_functions.sql index 840e33f1b..86a512a43 100644 --- a/packages/eql/src/v3/scalars/date/query_date_range_functions.sql +++ b/packages/eql/src/v3/scalars/date/query_date_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_date_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_date_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_date_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_date_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_date_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_date_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_date_ord`, `public.{{prefix}}_date_ord_ope`) diff --git a/packages/eql/src/v3/scalars/date/query_date_range_types.sql b/packages/eql/src/v3/scalars/date/query_date_range_types.sql index 31ec74c6a..0dcce4acc 100644 --- a/packages/eql/src/v3/scalars/date/query_date_range_types.sql +++ b/packages/eql/src/v3/scalars/date/query_date_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql b/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql index 8cdec0b7f..0c7bdd2a2 100644 --- a/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_double_ord_ore. --! @param a public.{{prefix}}_double_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_double_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_double_ord_ore. --! @param a public.{{prefix}}_double_ord_ore diff --git a/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql b/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql index e7e95f291..fe4943c20 100644 --- a/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_double_ord_ore. --! @param a {{prefix}}.query_double_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_double_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_double_ord_ore. --! @param a public.{{prefix}}_double_ord_ore diff --git a/packages/eql/src/v3/scalars/double/query_double_range_functions.sql b/packages/eql/src/v3/scalars/double/query_double_range_functions.sql index 8aa8ed9eb..d5282f6b2 100644 --- a/packages/eql/src/v3/scalars/double/query_double_range_functions.sql +++ b/packages/eql/src/v3/scalars/double/query_double_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_double_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_double_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_double_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_double_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_double_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_double_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_double_ord`, `public.{{prefix}}_double_ord_ope`) diff --git a/packages/eql/src/v3/scalars/double/query_double_range_types.sql b/packages/eql/src/v3/scalars/double/query_double_range_types.sql index 4939fa339..5603183ee 100644 --- a/packages/eql/src/v3/scalars/double/query_double_range_types.sql +++ b/packages/eql/src/v3/scalars/double/query_double_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/functions.sql b/packages/eql/src/v3/scalars/functions.sql index 1376a4719..affe0d755 100644 --- a/packages/eql/src/v3/scalars/functions.sql +++ b/packages/eql/src/v3/scalars/functions.sql @@ -58,3 +58,49 @@ BEGIN RAISE EXCEPTION 'operator % is not supported for %', operator_name, type_name; END; $$ LANGUAGE plpgsql; +-- {{#v4}} + +--! @brief Return a query operand unchanged when it carries the named term, +--! and raise when it does not. +--! +--! An EQL v4 search-domain query operand ({{prefix}}.query_text_search, +--! {{prefix}}.query_text_search_ore) carries only the terms its operator +--! reads, so each extractor over it reads its term through this helper. A +--! missing term raises here rather than reading as NULL, which would compare +--! as no row. +--! +--! A term whose value is JSON `null` is missing, and so is an empty array: a +--! `bf` with no positions is the filter of text too short for a 3-gram, which +--! a stored filter of the same text would contain, so a query carrying it +--! would match exactly the stored values no match query should find. +--! +--! The raise happens where the operand is evaluated, not where the statement +--! is planned. A literal operand is folded at plan time and raises even over +--! an empty table; a bound parameter under a generic plan raises wherever the +--! extractor reads it (an index scan always does, to build its key), and a +--! plan that never reaches the predicate returns no row and no error. +--! @internal +--! @param val jsonb The query operand. +--! @param domain_name text The operand's domain, for the message. +--! @param term text The JSON key of the term the extractor reads (hm, op, ob, bf). +--! @param operators text The operators that read the term, for the message. +--! @return jsonb `val`, unchanged. +--! @throws invalid_parameter_value When `val` has no `term` key, or its value +--! is JSON `null` or an empty array. The message names the domain, the +--! term and the operators, never the operand. +CREATE FUNCTION {{prefix}}_internal.require_query_term(val jsonb, domain_name text, term text, operators text) +RETURNS jsonb +IMMUTABLE STRICT PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + IF NOT (val ? term) + OR jsonb_typeof(val -> term) = 'null' + OR (jsonb_typeof(val -> term) = 'array' AND jsonb_array_length(val -> term) = 0) THEN + RAISE EXCEPTION '% value has no "%" term, the term % compare', domain_name, term, operators + USING ERRCODE = 'invalid_parameter_value'; + END IF; + RETURN val; +END; +$$ LANGUAGE plpgsql; +-- {{/v4}} diff --git a/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql b/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql index 4bfd5d2eb..cfbb1ecbd 100644 --- a/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_integer_ord_ore. --! @param a public.{{prefix}}_integer_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_integer_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_integer_ord_ore. --! @param a public.{{prefix}}_integer_ord_ore diff --git a/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql b/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql index 3578cfb1a..824fcb264 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_integer_ord_ore. --! @param a {{prefix}}.query_integer_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_integer_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_integer_ord_ore. --! @param a public.{{prefix}}_integer_ord_ore diff --git a/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql b/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql index d05e5c03a..68924936e 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_integer_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_integer_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_integer_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_integer_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_integer_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_integer_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_integer_ord`, `public.{{prefix}}_integer_ord_ope`) diff --git a/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql b/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql index c32c1be6c..5c9fe2cee 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql b/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql index 748793ea5..519252c7b 100644 --- a/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_numeric_ord_ore. --! @param a public.{{prefix}}_numeric_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_numeric_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_numeric_ord_ore. --! @param a public.{{prefix}}_numeric_ord_ore diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql index 092a57f9c..3426a4055 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_numeric_ord_ore. --! @param a {{prefix}}.query_numeric_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_numeric_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_numeric_ord_ore. --! @param a public.{{prefix}}_numeric_ord_ore diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql index 193f9763b..60ac2d189 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_numeric_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_numeric_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_numeric_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_numeric_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_numeric_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_numeric_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_numeric_ord`, `public.{{prefix}}_numeric_ord_ope`) diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql index 2bd28ed5b..2e47c91db 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/ore_fallback.sql b/packages/eql/src/v3/scalars/ore_fallback.sql index ce56a258a..6a699836f 100644 --- a/packages/eql/src/v3/scalars/ore_fallback.sql +++ b/packages/eql/src/v3/scalars/ore_fallback.sql @@ -25,7 +25,7 @@ --! --! Runs after the DO block in src/v3/sem/ore_block_256/operator_class.sql, --! which ATTEMPTS to create the default btree operator class for ---! {{prefix}}_internal.ore_block_256 and skips it on insufficient_privilege +--! {{prefix}}_internal.{{ore_block}} and skips it on insufficient_privilege --! (CREATE OPERATOR CLASS requires superuser; managed platforms — cloud --! Supabase and most hosted Postgres — run the installer as a non-superuser --! role). When the class was created, this file is a no-op. @@ -62,7 +62,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql b/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql index db512dd9a..40e483688 100644 --- a/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_real_ord_ore. --! @param a {{prefix}}.query_real_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_real_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_real_ord_ore. --! @param a public.{{prefix}}_real_ord_ore diff --git a/packages/eql/src/v3/scalars/real/query_real_range_functions.sql b/packages/eql/src/v3/scalars/real/query_real_range_functions.sql index 875251fd6..c1a74e2ca 100644 --- a/packages/eql/src/v3/scalars/real/query_real_range_functions.sql +++ b/packages/eql/src/v3/scalars/real/query_real_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_real_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_real_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_real_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_real_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_real_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_real_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_real_ord`, `public.{{prefix}}_real_ord_ope`) diff --git a/packages/eql/src/v3/scalars/real/query_real_range_types.sql b/packages/eql/src/v3/scalars/real/query_real_range_types.sql index b9508d140..e79669906 100644 --- a/packages/eql/src/v3/scalars/real/query_real_range_types.sql +++ b/packages/eql/src/v3/scalars/real/query_real_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql b/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql index c88e2b5b9..44da6bc7f 100644 --- a/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_real_ord_ore. --! @param a public.{{prefix}}_real_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_real_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_real_ord_ore. --! @param a public.{{prefix}}_real_ord_ore diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql index 8d898cf4a..db35558c1 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_smallint_ord_ore. --! @param a {{prefix}}.query_smallint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_smallint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_smallint_ord_ore. --! @param a public.{{prefix}}_smallint_ord_ore diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql index 2ec4f1809..df1dffbe9 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_smallint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_smallint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_smallint_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_smallint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_smallint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_smallint_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_smallint_ord`, `public.{{prefix}}_smallint_ord_ope`) diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql index 74ec90df4..bf467bef2 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql index 4940f690f..c01347c89 100644 --- a/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_smallint_ord_ore. --! @param a public.{{prefix}}_smallint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_smallint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_smallint_ord_ore. --! @param a public.{{prefix}}_smallint_ord_ore diff --git a/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql b/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql index f0f4b99f2..4295a7368 100644 --- a/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql @@ -16,11 +16,11 @@ AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; --! @brief Index extractor for {{prefix}}.query_text_ord_ore. --! @param a {{prefix}}.query_text_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_text_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_text_ord_ore. --! @param a public.{{prefix}}_text_ord_ore diff --git a/packages/eql/src/v3/scalars/text/query_text_range_functions.sql b/packages/eql/src/v3/scalars/text/query_text_range_functions.sql index 34735635d..02c5b9a9b 100644 --- a/packages/eql/src/v3/scalars/text/query_text_range_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_range_functions.sql @@ -23,14 +23,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_text_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_text_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_text_range_ore, as a block ORE term. --! @@ -40,14 +40,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_text_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_text_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_text_range_ope, as a CLLW OPE term. --! @@ -99,7 +99,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_text_ord`, `public.{{prefix}}_text_ord_ope`, `public.{{prefix}}_text_search`) @@ -134,7 +134,7 @@ AS $$ SELECT {{prefix}}.ord_term_ore(a) >= {{prefix}}_internal.lo(r) AND {{prefi --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_text_ord`, `public.{{prefix}}_text_ord_ope`, `public.{{prefix}}_text_search`) diff --git a/packages/eql/src/v3/scalars/text/query_text_range_types.sql b/packages/eql/src/v3/scalars/text/query_text_range_types.sql index 4b1fb707f..caaf084cf 100644 --- a/packages/eql/src/v3/scalars/text/query_text_range_types.sql +++ b/packages/eql/src/v3/scalars/text/query_text_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/text/query_text_search_functions.sql b/packages/eql/src/v3/scalars/text/query_text_search_functions.sql index 0bc6d563b..4b63185a4 100644 --- a/packages/eql/src/v3/scalars/text/query_text_search_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_search_functions.sql @@ -2,6 +2,7 @@ -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/scalars/text/query_text_types.sql -- REQUIRE: src/v3/scalars/text/text_search_functions.sql +-- REQUIRE: src/v3/scalars/functions.sql --! @file encrypted_domain/text/query_text_search_functions.sql --! @brief Functions for {{prefix}}.query_text_search. @@ -9,26 +10,50 @@ --! @brief Index extractor for {{prefix}}.query_text_search. --! @param a {{prefix}}.query_text_search --! @return {{prefix}}_internal.hmac_256 +-- {{#v4}} +--! @throws When the operand carries no `hm` term, the term = and <> compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.eq_term(a {{prefix}}.query_text_search) RETURNS {{prefix}}_internal.hmac_256 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.hmac_256({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search', 'hm', '= and <>')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search. --! @param a {{prefix}}.query_text_search --! @return {{prefix}}_internal.ope_cllw +-- {{#v4}} +--! @throws When the operand carries no `op` term, the term <, <=, > and >= compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.ord_term(a {{prefix}}.query_text_search) RETURNS {{prefix}}_internal.ope_cllw LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.ope_cllw(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.ope_cllw({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search', 'op', '<, <=, > and >=')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search. --! @param a {{prefix}}.query_text_search --! @return {{prefix}}_internal.bloom_filter +-- {{#v4}} +--! @throws When the operand carries no `bf` term, the term @@ compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.match_term(a {{prefix}}.query_text_search) RETURNS {{prefix}}_internal.bloom_filter LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.bloom_filter(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.bloom_filter({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search', 'bf', '@@')) $$; +-- {{/v4}} --! @brief Operator wrapper for {{prefix}}.query_text_search. --! @param a public.{{prefix}}_text_search diff --git a/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql b/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql index 4f1f4e94a..2815a05b3 100644 --- a/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql @@ -2,6 +2,7 @@ -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/scalars/text/query_text_types.sql -- REQUIRE: src/v3/scalars/text/text_search_ore_functions.sql +-- REQUIRE: src/v3/scalars/functions.sql --! @file encrypted_domain/text/query_text_search_ore_functions.sql --! @brief Functions for {{prefix}}.query_text_search_ore. @@ -9,26 +10,50 @@ --! @brief Index extractor for {{prefix}}.query_text_search_ore. --! @param a {{prefix}}.query_text_search_ore --! @return {{prefix}}_internal.hmac_256 +-- {{#v4}} +--! @throws When the operand carries no `hm` term, the term = and <> compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.eq_term(a {{prefix}}.query_text_search_ore) RETURNS {{prefix}}_internal.hmac_256 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.hmac_256({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'hm', '= and <>')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search_ore. --! @param a {{prefix}}.query_text_search_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} +-- {{#v4}} +--! @throws When the operand carries no `ob` term, the term <, <=, > and >= compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_text_search_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +-- {{#v3}} +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.{{ore_block}}({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'ob', '<, <=, > and >=')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search_ore. --! @param a {{prefix}}.query_text_search_ore --! @return {{prefix}}_internal.bloom_filter +-- {{#v4}} +--! @throws When the operand carries no `bf` term, the term @@ compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.match_term(a {{prefix}}.query_text_search_ore) RETURNS {{prefix}}_internal.bloom_filter LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.bloom_filter(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.bloom_filter({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'bf', '@@')) $$; +-- {{/v4}} --! @brief Operator wrapper for {{prefix}}.query_text_search_ore. --! @param a public.{{prefix}}_text_search_ore diff --git a/packages/eql/src/v3/scalars/text/query_text_types.sql b/packages/eql/src/v3/scalars/text/query_text_types.sql index 9360a9fbf..f6f8a9b1d 100644 --- a/packages/eql/src/v3/scalars/text/query_text_types.sql +++ b/packages/eql/src/v3/scalars/text/query_text_types.sql @@ -119,12 +119,23 @@ BEGIN jsonb_typeof(VALUE) = 'object' AND VALUE ? 'v' AND VALUE ? 'i' +-- {{#v3}} AND VALUE ? 'hm' AND VALUE ? 'ob' AND VALUE ? 'bf' +-- {{/v3}} +-- {{#v4}} + AND (VALUE ? 'hm' OR VALUE ? 'ob' OR VALUE ? 'bf') +-- {{/v4}} AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'array' + AND jsonb_array_length(VALUE -> 'ob') > 0)) +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -141,9 +152,14 @@ BEGIN jsonb_typeof(VALUE) = 'object' AND VALUE ? 'v' AND VALUE ? 'i' +-- {{#v3}} AND VALUE ? 'hm' AND VALUE ? 'op' AND VALUE ? 'bf' +-- {{/v3}} +-- {{#v4}} + AND (VALUE ? 'hm' OR VALUE ? 'op' OR VALUE ? 'bf') +-- {{/v4}} AND NOT (VALUE ? 'c') AND VALUE->>'v' = '{{eql_version}}' ); diff --git a/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql b/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql index d61d16612..74f3e200b 100644 --- a/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql @@ -19,11 +19,11 @@ AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; --! @brief Index extractor for public.{{prefix}}_text_ord_ore. --! @param a public.{{prefix}}_text_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_text_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_text_ord_ore. --! @param a public.{{prefix}}_text_ord_ore diff --git a/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql b/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql index fb3eaa35d..c15e31063 100644 --- a/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql @@ -20,11 +20,11 @@ AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; --! @brief Index extractor for public.{{prefix}}_text_search_ore. --! @param a public.{{prefix}}_text_search_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_text_search_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Index extractor for public.{{prefix}}_text_search_ore. --! @param a public.{{prefix}}_text_search_ore diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql index b081475a3..5b583178c 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_timestamp_ord_ore. --! @param a {{prefix}}.query_timestamp_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_timestamp_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_timestamp_ord_ore. --! @param a public.{{prefix}}_timestamp_ord_ore diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql index 6c1a95c89..cab511249 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_timestamp_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_timestamp_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_timestamp_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_timestamp_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_timestamp_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_timestamp_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_timestamp_ord`, `public.{{prefix}}_timestamp_ord_ope`) diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql index 795b8f267..02a882baa 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql b/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql index 6a83371f2..1727ad6da 100644 --- a/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_timestamp_ord_ore. --! @param a public.{{prefix}}_timestamp_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_timestamp_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_timestamp_ord_ore. --! @param a public.{{prefix}}_timestamp_ord_ore diff --git a/packages/eql/src/v3/schema.sql b/packages/eql/src/v3/schema.sql index d82dad656..bddd822bb 100644 --- a/packages/eql/src/v3/schema.sql +++ b/packages/eql/src/v3/schema.sql @@ -12,7 +12,7 @@ --! (Supabase/PostgREST calls functions, not operators) invoke them by name. --! {{prefix}}_internal houses INTERNAL implementation objects only: the --! searchable-encrypted-metadata (SEM) index-term types ---! ({{prefix}}_internal.hmac_256, {{prefix}}_internal.ore_block_256) and their support +--! ({{prefix}}_internal.hmac_256, {{prefix}}_internal.{{ore_block}}) and their support --! functions, the unsupported-operator blockers (which only raise), and the --! aggregate state functions. Together the two schemas are self-contained — --! they own every type they need and have no runtime dependency on another EQL diff --git a/packages/eql/src/v3/sem/bloom_filter/functions.sql b/packages/eql/src/v3/sem/bloom_filter/functions.sql index db4db5202..47e2295f7 100644 --- a/packages/eql/src/v3/sem/bloom_filter/functions.sql +++ b/packages/eql/src/v3/sem/bloom_filter/functions.sql @@ -16,7 +16,7 @@ --! @return boolean True when the `bf` key is present and non-null. --! --! @internal Defined for parity with the {{prefix}} SEM index-term predicates ---! (`has_hmac_256` / `has_ore_block_256`); it is not currently called by +--! (`has_hmac_256` / `has_{{ore_block}}`); it is not currently called by --! the extractor below, which gates on value-shape inline, nor by the generated --! domain CHECK, which tests `bf` presence via the envelope-key skeleton. Kept --! as the canonical presence test for callers that need one. diff --git a/packages/eql/src/v3/sem/ope_cllw/types.sql b/packages/eql/src/v3/sem/ope_cllw/types.sql index b27c193da..381696fc0 100644 --- a/packages/eql/src/v3/sem/ope_cllw/types.sql +++ b/packages/eql/src/v3/sem/ope_cllw/types.sql @@ -15,10 +15,10 @@ --! same pattern as {{prefix}}_internal.hmac_256 over text). That keeps the whole --! comparison chain inlinable, so a functional btree index on --! `{{prefix}}.ord_term(col)` engages structurally for the `_ord` / `_ord_ope` ---! domains' comparison operators. Contrast {{prefix}}_internal.ore_block_256 (`ob`), +--! domains' comparison operators. Contrast {{prefix}}_internal.{{ore_block}} (`ob`), --! the block-ORE term behind the `_ord_ore` escape hatch, compared by a custom --! N-block protocol. --! --! @note Transient type used only during query execution. ---! @see {{prefix}}_internal.ore_block_256 +--! @see {{prefix}}_internal.{{ore_block}} CREATE DOMAIN {{prefix}}_internal.ope_cllw AS bytea; diff --git a/packages/eql/src/v3/sem/ore_block_256/functions.sql b/packages/eql/src/v3/sem/ore_block_256/functions.sql index 1d6108e18..b77a4e48c 100644 --- a/packages/eql/src/v3/sem/ore_block_256/functions.sql +++ b/packages/eql/src/v3/sem/ore_block_256/functions.sql @@ -16,15 +16,15 @@ --! @brief Convert JSONB array to ORE block composite type --! @internal --! @param val jsonb Array of hex-encoded ORE block terms ---! @return {{prefix}}_internal.ore_block_256 ORE block composite, or NULL if input is null +--! @return {{prefix}}_internal.{{ore_block}} ORE block composite, or NULL if input is null --! @note plpgsql, not `LANGUAGE sql` (issue #353). The sole caller ---! (`ore_block_256`) is itself plpgsql, so this function is NEVER reached +--! (`{{ore_block}}`) is itself plpgsql, so this function is NEVER reached --! from an inlinable context — as `LANGUAGE sql` it paid the per-call --! SQL-function executor on every compared value in the opclass hot path --! (measured: +43% on ORE ordered scans vs the plpgsql form). The --! non-array guard preserves the v3 behaviour (returns NULL for a --! non-array scalar; the v2 plpgsql original raised); the caller only ---! reaches this when `has_ore_block_256(val)` is true, which requires +--! reaches this when `has_{{ore_block}}(val)` is true, which requires --! `val->'ob'` to be a JSON array, so that branch stays unreachable. --! An empty array (`ob: []`, what encrypting the empty string `""` produces) --! yields a non-NULL composite with an EMPTY `terms` array — NOT NULL terms. @@ -33,17 +33,17 @@ --! drops out of ordered queries). An empty array instead engages the --! comparator's `cardinality = 0` guard, which sorts empty BEFORE every --! non-empty term. See issue #262 (pinned by T7). -CREATE FUNCTION {{prefix}}_internal.jsonb_array_to_ore_block_256(val jsonb) -RETURNS {{prefix}}_internal.ore_block_256 +CREATE FUNCTION {{prefix}}_internal.jsonb_array_to_{{ore_block}}(val jsonb) +RETURNS {{prefix}}_internal.{{ore_block}} IMMUTABLE AS $$ DECLARE - terms {{prefix}}_internal.ore_block_256_term[]; + terms {{prefix}}_internal.{{ore_block}}_term[]; BEGIN IF val IS NULL OR jsonb_typeof(val) != 'array' THEN RETURN NULL; END IF; - SELECT array_agg(ROW(b)::{{prefix}}_internal.ore_block_256_term) + SELECT array_agg(ROW(b)::{{prefix}}_internal.{{ore_block}}_term) INTO terms FROM unnest({{prefix}}_internal.jsonb_array_to_bytea_array(val)) AS b; -- plpgsql pitfall: `SELECT INTO ` assigns the @@ -51,35 +51,35 @@ BEGIN -- constructor directly instead. The COALESCE stays load-bearing for the -- empty-`ob` case (issue #262): array_agg over zero rows yields NULL, and -- the comparator needs an EMPTY terms array, not NULL terms. - RETURN ROW(COALESCE(terms, ARRAY[]::{{prefix}}_internal.ore_block_256_term[]))::{{prefix}}_internal.ore_block_256; + RETURN ROW(COALESCE(terms, ARRAY[]::{{prefix}}_internal.{{ore_block}}_term[]))::{{prefix}}_internal.{{ore_block}}; END; $$ LANGUAGE plpgsql; --! @internal Keep the inline-critical marker so the post-install --! pin_search_path pass leaves this unpinned: a `SET search_path` clause on a --! plpgsql function forces per-call configuration switching — measurable on a ---! helper invoked per compared value in the ore_block_256 opclass hot path. +--! helper invoked per compared value in the {{ore_block}} opclass hot path. --! It takes a bare `jsonb` arg (not a jsonb-backed encrypted DOMAIN), so the --! structural skip in tasks/pin_search_path_v3.sql does not recognise it; --! this marker is the documented manual opt-in. -COMMENT ON FUNCTION {{prefix}}_internal.jsonb_array_to_ore_block_256(jsonb) IS +COMMENT ON FUNCTION {{prefix}}_internal.jsonb_array_to_{{ore_block}}(jsonb) IS 'eql-inline-critical: per-encrypted-value ORE opclass-path helper; must stay unpinned (SET search_path adds per-call overhead)'; --! @brief Extract ORE block index term from JSONB payload --! @param val jsonb containing encrypted EQL payload ---! @return {{prefix}}_internal.ore_block_256 ORE block index term +--! @return {{prefix}}_internal.{{ore_block}} ORE block index term --! @throws Exception if 'ob' field is missing -CREATE FUNCTION {{prefix}}_internal.ore_block_256(val jsonb) - RETURNS {{prefix}}_internal.ore_block_256 +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}(val jsonb) + RETURNS {{prefix}}_internal.{{ore_block}} IMMUTABLE STRICT PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN -- Declared STRICT: PostgreSQL returns NULL for a NULL argument without -- entering the body, so no explicit `val IS NULL` guard is needed. - IF {{prefix}}_internal.has_ore_block_256(val) THEN - RETURN {{prefix}}_internal.jsonb_array_to_ore_block_256(val->'ob'); + IF {{prefix}}_internal.has_{{ore_block}}(val) THEN + RETURN {{prefix}}_internal.jsonb_array_to_{{ore_block}}(val->'ob'); END IF; RAISE 'Expected an ore index (ob) value in json: %', val; END; @@ -91,13 +91,13 @@ $$ LANGUAGE plpgsql; --! @return boolean True only if the 'ob' field is present and is a JSON array --! @note A well-formed ORE index term is always a JSON array of block terms, so --! this guard treats a present-but-non-array `ob` (a scalar or object) as ---! absent. That makes the extractor `ore_block_256(val)` RAISE on a +--! absent. That makes the extractor `{{ore_block}}(val)` RAISE on a --! structurally invalid `ob` payload at the boundary instead of silently ---! degrading it to a NULL index term in `jsonb_array_to_ore_block_256`. The +--! degrading it to a NULL index term in `jsonb_array_to_{{ore_block}}`. The --! previous `val ->> 'ob' IS NOT NULL` form stringified scalars/objects and so --! reported them as present. `{}` (absent `ob`) and `{"ob": null}` (JSON null) --! both remain `false`. -CREATE FUNCTION {{prefix}}_internal.has_ore_block_256(val jsonb) +CREATE FUNCTION {{prefix}}_internal.has_{{ore_block}}(val jsonb) RETURNS boolean IMMUTABLE STRICT PARALLEL SAFE SET search_path = pg_catalog, extensions, public @@ -108,20 +108,21 @@ AS $$ $$ LANGUAGE plpgsql; +-- {{#v3}} --! @brief Compare two ORE block terms using cryptographic comparison --! @internal ---! @param a {{prefix}}_internal.ore_block_256_term First ORE term ---! @param b {{prefix}}_internal.ore_block_256_term Second ORE term +--! @param a {{prefix}}_internal.{{ore_block}}_term First ORE term +--! @param b {{prefix}}_internal.{{ore_block}}_term Second ORE term --! @return integer -1 if a < b, 0 if a = b, 1 if a > b --! @throws Exception if ciphertexts are different lengths ---! @note Marked `IMMUTABLE` (the three `compare_ore_block_256_term(s)` +--! @note Marked `IMMUTABLE` (the three `compare_{{ore_block}}_term(s)` --! overloads all are). This deliberately diverges from the v2 originals, --! which carry no volatility marker and so default to `VOLATILE`. The --! comparison is deterministic — its only crypto call, pgcrypto `encrypt()`, --! is itself `IMMUTABLE STRICT PARALLEL SAFE` — so `IMMUTABLE` lets the --! planner fold/cache these in ordering and index contexts. NOT `STRICT`: --! the NULL-handling branches below are load-bearing for the array overload. -CREATE FUNCTION {{prefix}}_internal.compare_ore_block_256_term(a {{prefix}}_internal.ore_block_256_term, b {{prefix}}_internal.ore_block_256_term) +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_term(a {{prefix}}_internal.{{ore_block}}_term, b {{prefix}}_internal.{{ore_block}}_term) RETURNS integer IMMUTABLE SET search_path = pg_catalog, extensions, public @@ -221,10 +222,10 @@ $$ LANGUAGE plpgsql; --! @brief Compare arrays of ORE block terms recursively --! @internal ---! @param a {{prefix}}_internal.ore_block_256_term[] First array ---! @param b {{prefix}}_internal.ore_block_256_term[] Second array +--! @param a {{prefix}}_internal.{{ore_block}}_term[] First array +--! @param b {{prefix}}_internal.{{ore_block}}_term[] Second array --! @return integer -1/0/1, or NULL if either array is NULL -CREATE FUNCTION {{prefix}}_internal.compare_ore_block_256_terms(a {{prefix}}_internal.ore_block_256_term[], b {{prefix}}_internal.ore_block_256_term[]) +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}_term[], b {{prefix}}_internal.{{ore_block}}_term[]) RETURNS integer IMMUTABLE SET search_path = pg_catalog, extensions, public @@ -248,28 +249,269 @@ AS $$ RETURN 1; END IF; - cmp_result := {{prefix}}_internal.compare_ore_block_256_term(a[1], b[1]); + cmp_result := {{prefix}}_internal.compare_{{ore_block}}_term(a[1], b[1]); IF cmp_result = 0 THEN - RETURN {{prefix}}_internal.compare_ore_block_256_terms(a[2:array_length(a,1)], b[2:array_length(b,1)]); + RETURN {{prefix}}_internal.compare_{{ore_block}}_terms(a[2:array_length(a,1)], b[2:array_length(b,1)]); END IF; RETURN cmp_result; END $$ LANGUAGE plpgsql; +-- {{/v3}} +-- {{#v4}} +--! @brief The 1-bit hash of a 6-bit block ORE comparison +--! @internal +--! @param f bytea The 16-byte left tag of the first differing block +--! @param nonce bytea The 16-byte nonce of the right-hand ciphertext +--! @return integer 0 or 1 +--! @note The BHKR sigma-MMO hash ore-rs's 6-bit schemes use: +--! `m = 2*f XOR nonce` (doubling in GF(2^128), big-endian, reduction +--! constant 0x87), then the low bit of byte 0 of `AES(PI_KEY, m) XOR m`, +--! where `PI_KEY` is the public, fixed AES key `ORE-rs.v2.H-pi.1`. The +--! doubling is done on two `bigint` halves with no branch on the data. Its +--! vectors are ore.rs's own (`tests/encryption/tests/v4_ore_block.rs`). +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_hash_bit(f bytea, nonce bytea) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE + SET search_path = pg_catalog, extensions, public +AS $$ + DECLARE + hi bigint := ('x' || encode(substr(f, 1, 8), 'hex'))::bit(64)::bigint; + lo bigint := ('x' || encode(substr(f, 9, 8), 'hex'))::bit(64)::bigint; + m bytea; + BEGIN + -- 2*f: shift the 128 bits left by one, carrying lo's top bit into hi, + -- and fold hi's top bit back in as 0x87. Then XOR the nonce. + m := int8send(((hi << 1) | (lo < 0)::int) + # ('x' || encode(substr(nonce, 1, 8), 'hex'))::bit(64)::bigint) + || int8send(((lo << 1) # (135 * (hi < 0)::int)) + # ('x' || encode(substr(nonce, 9, 8), 'hex'))::bit(64)::bigint); + RETURN get_bit(encrypt(m, 'ORE-rs.v2.H-pi.1'::bytea, 'aes-ecb/pad:none'), 0) + # get_bit(m, 0); + END; +$$ LANGUAGE plpgsql; + + +--! @brief The scheme of a 6-bit block ORE term, checking its shape +--! @internal +--! @param t bytea A serialised ore-rs ciphertext +--! @return integer 2 for the fixed-width 6-bit scheme, 3 for the chained one +--! @throws Exception if `t` is not a well-formed ciphertext of either: a +--! header of version 2, a block count of at least one (at most 14 for the +--! fixed-width scheme, at most 1366 for the chained one), a length of +--! exactly header + 25 bytes a block + a 16-byte nonce, a padding policy +--! the chained scheme writes with a block count it can produce, and every +--! permuted symbol below 64. The message names the length only, never the +--! bytes. +--! @note The checks ore-rs's comparators make before they compare; every +--! one is on the term's public structure. All of them are expressions, +--! with no sub-select, because this runs on both operands of every +--! comparison. +--! @note The chained bound is tighter than the wire format's 65,535 blocks. +--! 1366 blocks is a 1,024-byte string, the longest Stack Encrypt writes +--! (ore-rs's default ceiling, which counts the marker byte before text +--! and bytes). Comparing a term costs time in its block count, so a +--! stored term past the bound would slow every comparison against it. +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_scheme(t bytea) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE + SET search_path = pg_catalog, extensions, public +AS $$ + DECLARE + len integer := octet_length(t); + scheme integer; + header integer; + count integer; + max_count integer; + kind integer; + policy integer; + bucket integer; + admitted boolean; + BEGIN + IF len < 7 OR get_byte(t, 0) <> 2 OR get_byte(t, 1) NOT IN (2, 3) THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + scheme := get_byte(t, 1); + header := CASE scheme WHEN 2 THEN 4 ELSE 7 END; + count := get_byte(t, 2) * 256 + get_byte(t, 3); + max_count := CASE scheme WHEN 2 THEN 14 ELSE 1366 END; + IF count < 1 OR count > max_count OR len <> header + 25 * count + 16 THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + IF scheme = 3 THEN + -- The padding policy: none (0, 0), powers of two from a floor (1, n) + -- or one fixed length (2, n), and a block count it can produce. A + -- string of n bytes is ceil(8n / 6) blocks, which grows with n, so + -- the one length that could give `count` blocks is floor(6 count / 8). + kind := get_byte(t, 4); + policy := get_byte(t, 5) * 256 + get_byte(t, 6); + bucket := 6 * count / 8; + admitted := CASE kind + WHEN 0 THEN policy = 0 + WHEN 1 THEN policy <= 49151 AND ( + count = (8 * policy + 5) / 6 + OR (bucket > policy AND bucket & (bucket - 1) = 0 + AND count = (8 * bucket + 5) / 6)) + WHEN 2 THEN policy <= 49151 AND count = (8 * policy + 5) / 6 + ELSE false + END; + IF NOT admitted THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + END IF; + -- A byte is below 64 exactly when its high hex digit is 0 to 3. + IF encode(substr(t, header + 1, count), 'hex') !~ '^([0-3][0-9a-f])*$' THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + RETURN scheme; + END; +$$ LANGUAGE plpgsql; + + +--! @brief Compare two 6-bit block ORE terms +--! @internal +--! @param a {{prefix}}_internal.{{ore_block}}_term First term: its left half is read +--! @param b {{prefix}}_internal.{{ore_block}}_term Second term: its right half is read +--! @return integer -1 if a < b, 0 if a = b, 1 if a > b +--! @throws Exception if either term is malformed +--! (`{{prefix}}_internal.{{ore_block}}_scheme`), if the two are of different +--! schemes, if two fixed-width terms have different block counts, or if +--! two chained terms were padded under different policies. A mismatched +--! pair raises rather than comparing to a meaningless answer. +--! @note Stack Encrypt writes ore-rs's 6-bit block ORE (ADR-0002): the +--! fixed-width scheme (wire header `02 02`) for integers and floats, and +--! the chained scheme (`02 03`, then a padding policy) for text. Both are +--! ore-rs's wire version 2, which names the type here (`{{ore_block}}`); +--! EQL v3's type of the same role compares cipherstash-client's 8-bit +--! blocks, a different format under the same `ob` key. +--! The layout after the header, for `count` blocks: `count` permuted +--! symbols, `count` 16-byte left tags, a 16-byte nonce, then `count` +--! 8-byte right blocks. Chained terms compare lexicographically: if the +--! shorter one's blocks all match, the shorter sorts first. +--! @note Constant operation count for operands of one shape (scheme and +--! block counts, which a term's length reveals anyway). One statement per +--! compared block evaluates both of its comparisons (integer `|`, which +--! does not short-circuit) and latches the first differing block with +--! arithmetic rather than an `IF`. The hash runs exactly once on every +--! path, equal terms included (on block 0, its result discarded), and the +--! result is selected arithmetically. PL/pgSQL is not a constant-time +--! environment: C-level effects remain (`memcmp` stopping inside 16 bytes, +--! the offset `substr` reads at). See `tests/timing/ore_block_v2/`. +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_term(a {{prefix}}_internal.{{ore_block}}_term, b {{prefix}}_internal.{{ore_block}}_term) + RETURNS integer + IMMUTABLE + SET search_path = pg_catalog, extensions, public +AS $$ + DECLARE + ab bytea; + bb bytea; + scheme integer; + header integer; + ca integer; -- a's block count + cb integer; -- b's block count + nonce_b integer; -- 0-based offset of b's nonce; its right blocks follow + latch integer := 0; -- 1 + the first differing block, 0 while none differs + first_diff integer; + indicator integer; + BEGIN + IF a IS NULL AND b IS NULL THEN + RETURN 0; + END IF; + + IF a IS NULL THEN + RETURN -1; + END IF; + + IF b IS NULL THEN + RETURN 1; + END IF; + + ab := a.bytes; + bb := b.bytes; + scheme := {{prefix}}_internal.{{ore_block}}_scheme(ab); + IF {{prefix}}_internal.{{ore_block}}_scheme(bb) <> scheme THEN + RAISE EXCEPTION 'ORE block terms of different schemes'; + END IF; + header := CASE scheme WHEN 2 THEN 4 ELSE 7 END; + ca := get_byte(ab, 2) * 256 + get_byte(ab, 3); + cb := get_byte(bb, 2) * 256 + get_byte(bb, 3); + IF scheme = 2 AND ca <> cb THEN + RAISE EXCEPTION 'Ciphertexts are different lengths'; + END IF; + IF scheme = 3 AND substr(ab, 5, 3) <> substr(bb, 5, 3) THEN + RAISE EXCEPTION 'ORE block terms padded under different policies'; + END IF; + nonce_b := header + 17 * cb; + + -- One statement per block, the same for every block. + FOR block IN 0..least(ca, cb) - 1 LOOP + latch := latch + (latch = 0)::int * (block + 1) * ( + (get_byte(ab, header + block) <> get_byte(bb, header + block))::int + | (substr(ab, header + ca + 16 * block + 1, 16) + <> substr(bb, header + cb + 16 * block + 1, 16))::int); + END LOOP; + + -- Equal prefixes read block 0. + first_diff := greatest(latch - 1, 0); + + -- One bit read and one hash on every path: bit `xt` of b's right block, + -- XOR the hash of a's left tag under b's nonce. 1 means a > b. + indicator := get_bit( + substr(bb, nonce_b + 17 + 8 * first_diff, 8), + get_byte(ab, header + first_diff)) + # {{prefix}}_internal.{{ore_block}}_hash_bit( + substr(ab, header + ca + 16 * first_diff + 1, 16), + substr(bb, nonce_b + 1, 16)); + + -- No difference: the shorter sorts first (equal lengths are equal). + -- Otherwise the indicator. No branch on the data. + RETURN (latch = 0)::int * sign(ca - cb)::integer + + (latch > 0)::int * (2 * indicator - 1); + END; +$$ LANGUAGE plpgsql; + + +--! @brief Compare two EQL v4 ORE block values +--! @internal +--! @param a {{prefix}}_internal.{{ore_block}}_term[] First value's terms +--! @param b {{prefix}}_internal.{{ore_block}}_term[] Second value's terms +--! @return integer -1/0/1, or NULL if either array is NULL +--! @throws Exception unless each value holds exactly one term: Stack +--! Encrypt writes one 6-bit block ORE term per value, text included, where +--! EQL v3's text held up to six 8-bit terms +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}_term[], b {{prefix}}_internal.{{ore_block}}_term[]) +RETURNS integer + IMMUTABLE + SET search_path = pg_catalog, extensions, public +AS $$ + BEGIN + IF a IS NULL OR b IS NULL THEN + RETURN NULL; + END IF; + + IF cardinality(a) <> 1 OR cardinality(b) <> 1 THEN + RAISE EXCEPTION 'An ORE block value holds exactly one term, not % and %', + cardinality(a), cardinality(b); + END IF; + + RETURN {{prefix}}_internal.compare_{{ore_block}}_term(a[1], b[1]); + END +$$ LANGUAGE plpgsql; +-- {{/v4}} --! @brief Compare ORE block composite types --! @internal ---! @param a {{prefix}}_internal.ore_block_256 First ORE block ---! @param b {{prefix}}_internal.ore_block_256 Second ORE block +--! @param a {{prefix}}_internal.{{ore_block}} First ORE block +--! @param b {{prefix}}_internal.{{ore_block}} Second ORE block --! @return integer -1/0/1 -CREATE FUNCTION {{prefix}}_internal.compare_ore_block_256_terms(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS integer IMMUTABLE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.compare_ore_block_256_terms(a.terms, b.terms); + RETURN {{prefix}}_internal.compare_{{ore_block}}_terms(a.terms, b.terms); END $$ LANGUAGE plpgsql; diff --git a/packages/eql/src/v3/sem/ore_block_256/operator_class.sql b/packages/eql/src/v3/sem/ore_block_256/operator_class.sql index b889c5cd9..c2a5f1bf6 100644 --- a/packages/eql/src/v3/sem/ore_block_256/operator_class.sql +++ b/packages/eql/src/v3/sem/ore_block_256/operator_class.sql @@ -4,7 +4,7 @@ -- REQUIRE: src/v3/sem/ore_block_256/operators.sql --! @file v3/sem/ore_block_256/operator_class.sql ---! @brief B-tree operator family + default class on {{prefix}}_internal.ore_block_256. +--! @brief B-tree operator family + default class on {{prefix}}_internal.{{ore_block}}. --! --! Gives the composite type its DEFAULT btree opclass so the recommended --! functional index `CREATE INDEX ON t ({{prefix}}.ord_term_ore(col))` engages without @@ -16,32 +16,32 @@ --! the installer as a non-superuser role, so the DO block below ATTEMPTS --! the creation and skips it on insufficient_privilege (SQLSTATE 42501), --! letting the single installer run everywhere. When the class is absent, ---! ORE ordered scans over {{prefix}}_internal.ore_block_256 are unavailable, +--! ORE ordered scans over {{prefix}}_internal.{{ore_block}} are unavailable, --! but the order-preserving (OPE) ordering domains — whose extractor --! return types carry a native btree opclass — still index without it. On --! superuser installs (self-managed Postgres, the SQLx test matrix) the --! class is created normally. Any non-privilege error still propagates. ---! @see {{prefix}}_internal.compare_ore_block_256_terms +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms DO $do$ BEGIN - EXECUTE 'CREATE OPERATOR FAMILY {{prefix}}_internal.ore_block_256_operator_family USING btree'; + EXECUTE 'CREATE OPERATOR FAMILY {{prefix}}_internal.{{ore_block}}_operator_family USING btree'; EXECUTE $ddl$ - CREATE OPERATOR CLASS {{prefix}}_internal.ore_block_256_operator_class - DEFAULT FOR TYPE {{prefix}}_internal.ore_block_256 - USING btree FAMILY {{prefix}}_internal.ore_block_256_operator_family AS + CREATE OPERATOR CLASS {{prefix}}_internal.{{ore_block}}_operator_class + DEFAULT FOR TYPE {{prefix}}_internal.{{ore_block}} + USING btree FAMILY {{prefix}}_internal.{{ore_block}}_operator_family AS OPERATOR 1 public.<, OPERATOR 2 public.<=, OPERATOR 3 public.=, OPERATOR 4 public.>=, OPERATOR 5 public.>, - FUNCTION 1 {{prefix}}_internal.compare_ore_block_256_terms(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) + FUNCTION 1 {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) $ddl$; - RAISE NOTICE 'EQL: created btree operator class {{prefix}}_internal.ore_block_256_operator_class'; + RAISE NOTICE 'EQL: created btree operator class {{prefix}}_internal.{{ore_block}}_operator_class'; EXCEPTION WHEN insufficient_privilege THEN - RAISE NOTICE 'EQL: skipped operator class {{prefix}}_internal.ore_block_256_operator_class (requires superuser); ORE ordered indexes on ore_block_256 unavailable, OPE ordering domains unaffected'; + RAISE NOTICE 'EQL: skipped operator class {{prefix}}_internal.{{ore_block}}_operator_class (requires superuser); ORE ordered indexes on {{ore_block}} unavailable, OPE ordering domains unaffected'; END; $do$; diff --git a/packages/eql/src/v3/sem/ore_block_256/operators.sql b/packages/eql/src/v3/sem/ore_block_256/operators.sql index cdd4c9df3..8a72f788f 100644 --- a/packages/eql/src/v3/sem/ore_block_256/operators.sql +++ b/packages/eql/src/v3/sem/ore_block_256/operators.sql @@ -3,7 +3,7 @@ -- REQUIRE: src/v3/sem/ore_block_256/functions.sql --! @file v3/sem/ore_block_256/operators.sql ---! @brief Comparison operators on {{prefix}}_internal.ore_block_256. +--! @brief Comparison operators on {{prefix}}_internal.{{ore_block}}. --! --! The six backing functions are inlinable single-statement SQL so the planner --! can fold the {{prefix}} comparison wrappers through to functional-index matching. @@ -11,97 +11,97 @@ --! @brief Equality backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the ORE blocks are equal --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_eq(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_eq(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) = 0 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) = 0 $$; --! @brief Not-equal backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the ORE blocks are not equal --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_neq(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_neq(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) <> 0 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) <> 0 $$; --! @brief Less-than backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is less than the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_lt(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_lt(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) = -1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) = -1 $$; --! @brief Less-than-or-equal backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is less than or equal to the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_lte(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_lte(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) != 1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) != 1 $$; --! @brief Greater-than backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is greater than the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_gt(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_gt(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) = 1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) = 1 $$; --! @brief Greater-than-or-equal backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is greater than or equal to the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_gte(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_gte(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) != -1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) != -1 $$; @@ -112,9 +112,9 @@ $$; --! first time an ore_block equality is used as a join qual (e.g. via the inlined --! {{prefix}}_internal._ord_ore equality wrappers). CREATE OPERATOR public.= ( - FUNCTION={{prefix}}_internal.ore_block_256_eq, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_eq, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.=), NEGATOR = OPERATOR(public.<>), RESTRICT = eqsel, @@ -125,9 +125,9 @@ CREATE OPERATOR public.= ( --! @brief <> operator for ORE block types CREATE OPERATOR public.<> ( - FUNCTION={{prefix}}_internal.ore_block_256_neq, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_neq, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.<>), NEGATOR = OPERATOR(public.=), RESTRICT = neqsel, @@ -137,9 +137,9 @@ CREATE OPERATOR public.<> ( --! @brief > operator for ORE block types CREATE OPERATOR public.> ( - FUNCTION={{prefix}}_internal.ore_block_256_gt, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_gt, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.<), NEGATOR = OPERATOR(public.<=), RESTRICT = scalargtsel, @@ -148,9 +148,9 @@ CREATE OPERATOR public.> ( --! @brief < operator for ORE block types CREATE OPERATOR public.< ( - FUNCTION={{prefix}}_internal.ore_block_256_lt, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_lt, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.>), NEGATOR = OPERATOR(public.>=), RESTRICT = scalarltsel, @@ -159,9 +159,9 @@ CREATE OPERATOR public.< ( --! @brief <= operator for ORE block types CREATE OPERATOR public.<= ( - FUNCTION={{prefix}}_internal.ore_block_256_lte, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_lte, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.>=), NEGATOR = OPERATOR(public.>), RESTRICT = scalarlesel, @@ -170,9 +170,9 @@ CREATE OPERATOR public.<= ( --! @brief >= operator for ORE block types CREATE OPERATOR public.>= ( - FUNCTION={{prefix}}_internal.ore_block_256_gte, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_gte, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.<=), NEGATOR = OPERATOR(public.<), RESTRICT = scalargesel, diff --git a/packages/eql/src/v3/sem/ore_block_256/types.sql b/packages/eql/src/v3/sem/ore_block_256/types.sql index b697ddaf6..f9996c611 100644 --- a/packages/eql/src/v3/sem/ore_block_256/types.sql +++ b/packages/eql/src/v3/sem/ore_block_256/types.sql @@ -10,7 +10,7 @@ --! --! Composite type representing a single ORE block term. Stores encrypted data --! as bytea that enables range comparisons without decryption. -CREATE TYPE {{prefix}}_internal.ore_block_256_term AS ( +CREATE TYPE {{prefix}}_internal.{{ore_block}}_term AS ( bytes bytea ); @@ -21,6 +21,6 @@ CREATE TYPE {{prefix}}_internal.ore_block_256_term AS ( --! in the 'ob' field of encrypted data payloads. --! --! @note Transient type used only during query execution. -CREATE TYPE {{prefix}}_internal.ore_block_256 AS ( - terms {{prefix}}_internal.ore_block_256_term[] +CREATE TYPE {{prefix}}_internal.{{ore_block}} AS ( + terms {{prefix}}_internal.{{ore_block}}_term[] ); diff --git a/packages/eql/tasks/pin_search_path_v3.sql b/packages/eql/tasks/pin_search_path_v3.sql index 8b5576617..593c6a699 100644 --- a/packages/eql/tasks/pin_search_path_v3.sql +++ b/packages/eql/tasks/pin_search_path_v3.sql @@ -12,7 +12,7 @@ --! satisfying Supabase splinter's `function_search_path_mutable` lint. --! --! @note A SET clause disables SQL-function inlining. The inline-critical SEM ---! helpers (ore_block_256_*, ope_cllw, hmac_256, bloom_filter over +--! helpers ({{ore_block}}_*, ope_cllw, hmac_256, bloom_filter over --! jsonb) and the encrypted-domain family (recognised structurally, --! including public user-column domains) are deliberately left --! unpinned. @@ -43,9 +43,9 @@ BEGIN WHERE n.nspname = ANY({{prefix}}_internal.owned_schemas()) AND ( (p.pronargs = 2 - AND p.proname IN ('ore_block_256_eq', 'ore_block_256_neq', - 'ore_block_256_lt', 'ore_block_256_lte', - 'ore_block_256_gt', 'ore_block_256_gte')) + AND p.proname IN ('{{ore_block}}_eq', '{{ore_block}}_neq', + '{{ore_block}}_lt', '{{ore_block}}_lte', + '{{ore_block}}_gt', '{{ore_block}}_gte')) -- The CLLW-OPE surface is the extractor alone: {{prefix}}_internal.ope_cllw is a -- domain over bytea (native comparison operators and btree opclass), -- so there are no ope-specific comparison functions to keep inlinable. diff --git a/packages/eql/tests/encryption/Cargo.toml b/packages/eql/tests/encryption/Cargo.toml index 7030eead1..6ee6ec13c 100644 --- a/packages/eql/tests/encryption/Cargo.toml +++ b/packages/eql/tests/encryption/Cargo.toml @@ -11,6 +11,12 @@ name = "text_eq_example" path = "../../crates/eql-bindings/src/encryption/example.rs" harness = false +# The same, for the typed query example in `eql_bindings::encryption::query`. +[[test]] +name = "query_example" +path = "../../crates/eql-bindings/src/encryption/query_example.rs" +harness = false + # Keep crypto/Postgres test dependencies out of eql-bindings' default test build. [dev-dependencies] # `chrono` and `rust_decimal`: the date, timestamp and decimal plaintexts @@ -47,10 +53,6 @@ postgres = "0.19" # crates protect-ffi ships. Primitive dependencies such as blake3 are not # compared. # -# The EQL v4 range tests (tests/v4_range.rs) use it too: EQL compares -# block-ORE and CLLW-OPE terms, which Stack Encrypt does not derive for EQL -# yet, so cipherstash-client's `OreIndexer` / `OpeIndexer` write the order -# terms of their rows and ranges, offline over the same index key. cipherstash-client = "=0.42.3" # The tampering test decodes a stored body, changes one byte and re-encodes # it. The same line eql-bindings uses, so the lockfile gains no second copy. diff --git a/packages/eql/tests/encryption/tests/common/mod.rs b/packages/eql/tests/encryption/tests/common/mod.rs index e64883b5c..634369030 100644 --- a/packages/eql/tests/encryption/tests/common/mod.rs +++ b/packages/eql/tests/encryption/tests/common/mod.rs @@ -12,6 +12,9 @@ use uuid::Uuid; pub struct Calls { pub generate: Vec>, pub retrieve: Vec>, + /// Index key loads, one per keyset a call resolves after the cipher is + /// built. + pub loads: usize, } pub struct Observed { @@ -51,6 +54,7 @@ impl IndexKeySource for Observed { &self, keyset: Option, ) -> Result<(Uuid, IndexKey), stack_kms::Error> { + self.calls.lock().unwrap().loads += 1; self.inner.load_index_key(keyset).await } } @@ -64,5 +68,20 @@ pub async fn cipher() -> (StackCipher, Arc>) { .init() .await .unwrap(); + calls.lock().unwrap().loads = 0; (cipher, calls) } + +/// The index a query on the named type is asked through when the test does +/// not care which: the first the type carries. Equality for a name with no +/// row or no index, which a test of a refusal names; the refusal comes +/// before the index is read. +// Not every suite that includes this module queries by name. +#[allow(dead_code)] +pub fn index_of(name: &str) -> stack_encrypt::target::IndexSpec { + use stack_encrypt::target::IndexSpec; + eql_bindings::encryption::targets::target(name) + .and_then(|target| target.indexes.first()) + .and_then(|key| IndexSpec::parse(key)) + .unwrap_or(IndexSpec::Equality) +} diff --git a/packages/eql/tests/encryption/tests/heap_hygiene.rs b/packages/eql/tests/encryption/tests/heap_hygiene.rs index 32b553e3b..ef7818cbe 100644 --- a/packages/eql/tests/encryption/tests/heap_hygiene.rs +++ b/packages/eql/tests/encryption/tests/heap_hygiene.rs @@ -11,9 +11,10 @@ //! //! One free is excluded by name: vitaminc-encrypt 0.6.0's aws-lc backend //! (`backend::aws_lc::CipherKey::seal`, native builds only) frees an unwiped -//! plaintext buffer on every seal. That is vitaminc's to fix; the spy -//! recognises those frees by their backtrace, so a build without symbols -//! fails rather than passes. +//! plaintext buffer on every seal; that is vitaminc's to fix. It is counted +//! and printed, never silent, and every other free on those paths is still +//! watched. The spy recognises these frees by their backtrace, so a build +//! without symbols fails rather than passes. //! //! Unix only: the spy scans a freed block with libc's `memmem`, not as a @@ -42,12 +43,16 @@ thread_local! { /// threads, and the findings below are one set. static GATE: Mutex<()> = Mutex::new(()); -/// The marked frees seen, as backtraces; and how many were vitaminc's. +/// The marked frees seen, as backtraces; and how many each exclusion took. static FOUND: Mutex> = Mutex::new(Vec::new()); -static UPSTREAM: Mutex = Mutex::new(0); +static EXCLUDED: Mutex<[usize; 1]> = Mutex::new([0; 1]); -/// The vitaminc frame whose free is excluded (see the module docs). -const UPSTREAM_SEAL: &str = "vitaminc_encrypt::backend::aws_lc::CipherKey::seal"; +/// The frames whose frees are excluded, each with what it is (see the module +/// docs): vitaminc's aws-lc seal. +const EXCLUSIONS: [(&str, &str); 1] = [( + "vitaminc_encrypt::backend::aws_lc::CipherKey::seal", + "vitaminc's aws-lc seal", +)]; /// Whether the `len` bytes at `ptr` hold [`MARK`]. A freed block can hold /// bytes Rust never initialised (a `Vec`'s spare capacity, a struct's @@ -87,8 +92,12 @@ unsafe impl GlobalAlloc for Spy { // Symbol demangling differs by platform: Linux writes // `<…::CipherKey>::seal`, macOS `…::CipherKey::seal`. Match // with the angle brackets dropped. - if trace.replace(['<', '>'], "").contains(UPSTREAM_SEAL) { - *UPSTREAM.lock().unwrap() += 1; + let plain = trace.replace(['<', '>'], ""); + if let Some(at) = EXCLUSIONS + .iter() + .position(|(frame, _)| plain.contains(frame)) + { + EXCLUDED.lock().unwrap()[at] += 1; } else { FOUND .lock() @@ -110,14 +119,16 @@ static SPY: Spy = Spy; fn watched(what: &str, f: impl FnOnce() -> T) -> T { let _gate = GATE.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); FOUND.lock().unwrap().clear(); - *UPSTREAM.lock().unwrap() = 0; + *EXCLUDED.lock().unwrap() = [0; 1]; WATCHING.with(|on| on.set(true)); let out = f(); WATCHING.with(|on| on.set(false)); let found = std::mem::take(&mut *FOUND.lock().unwrap()); - let upstream = std::mem::take(&mut *UPSTREAM.lock().unwrap()); - if upstream > 0 { - eprintln!("{what}: skipped {upstream} unwiped frees in vitaminc's aws-lc seal"); + let excluded = std::mem::take(&mut *EXCLUDED.lock().unwrap()); + for ((_, name), skipped) in EXCLUSIONS.iter().zip(excluded) { + if skipped > 0 { + eprintln!("{what}: skipped {skipped} unwiped frees in {name}"); + } } assert!( found.is_empty(), @@ -136,14 +147,37 @@ fn text(value: &str) -> ValueFrame { ValueFrame::of(value).unwrap() } +/// Every index the target carries, each of which a query may be asked +/// through. A type whose query is built per index (`TextSearch`) runs a +/// different projection, and its conversion into the query twin, for each. +fn indexes_of(target: &targets::Target) -> Vec { + target + .indexes + .iter() + .map(|key| stack_encrypt::target::IndexSpec::parse(key).expect("an engine index key")) + .collect() +} + fn runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_current_thread() .build() .unwrap() } -/// The producible EQL types over text, each with whether it has a query. -const TEXT_TARGETS: [(&str, bool); 3] = [("Text", false), ("TextEq", true), ("TextMatch", true)]; +/// The producible EQL types over text, read from the target table so a type +/// that becomes producible is watched without an edit here. +fn text_targets() -> Vec<&'static targets::Target> { + let found: Vec<_> = targets::targets() + .iter() + .filter(|target| target.family == "text" && target.producible) + .collect(); + assert!( + found.iter().any(|target| target.query.is_none()) + && found.iter().any(|target| target.query.is_some()), + "the table lists a storage-only and a queried text type" + ); + found +} #[test] fn a_text_target_leaves_no_plaintext_on_the_heap() { @@ -157,7 +191,8 @@ fn a_text_target_leaves_no_plaintext_on_the_heap() { }); let keyset = cipher.default_keyset(); let label = Label::new(["users", "email"]).unwrap(); - for (name, has_query) in TEXT_TARGETS { + for target in text_targets() { + let name = target.name; let plaintext = text(&marked("stored")); let stored = watched(&format!("{name} encrypt"), || { rt.block_on(async { @@ -189,11 +224,14 @@ fn a_text_target_leaves_no_plaintext_on_the_heap() { }); drop(opened); }); - if has_query { + if target.query.is_none() { + continue; + } + for index in indexes_of(target) { let query = text(&marked("query")); - watched(&format!("{name} query"), || { + watched(&format!("{name} query through {}", index.key()), || { rt.block_on(async { - targets::query(name, &keyset, &label, query) + targets::query(name, &index, &keyset, &label, query) .unwrap() .await .unwrap() diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs new file mode 100644 index 000000000..dd7484643 --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -0,0 +1,703 @@ +//! The JSON tree family end to end: a document sealed by Stack Encrypt's +//! JSON index into `public.eql_v4_json_tree`, queried with the v4 SQL, and +//! opened again. The unit tests need no database; the PostgreSQL test +//! installs nothing itself (`mise run test:encryption:postgres` installs the +//! v4 bundle built from the tree) and is the oracle the design asks for: +//! `@>` and `<@` over the encrypted column against PostgreSQL's own +//! `jsonb @>` over the plaintexts, for the design's corpus and random +//! documents, with a GIN index on the projection that the planner is shown +//! to use. + +mod common; + +use eql_bindings::v4::json_tree::{ + JsonTreeDocument, JsonTreeEntry, JsonTreeLeafQuery, JsonTreeQuery, JsonTreeSubtree, +}; +use eql_bindings::Identifier; +use serde_json::{json, Map, Value}; +use stack_encrypt::json::{JsonDocument, JsonOptions, JsonPath}; +use stack_encrypt::plan::{Typed, ValuePlan}; +use stack_encrypt::{op, NonEmpty, Plan}; + +fn column() -> NonEmpty { + Identifier::for_column("users", "profile").unwrap() +} + +fn fixture() -> Value { + json!({"hello": "world-1", "nested": {"deep": "constant", "list": [{"x": 1}, {"x": 2}]}, "number": 1}) +} + +type Cipher = stack_encrypt::StackCipher; + +async fn seal(cipher: &Cipher, value: &Value) -> JsonTreeDocument { + cipher + .default_keyset() + .encrypt_as::(value, column()) + .await + .unwrap() +} + +/// The plan a query derives through: the engine's own document target +/// under the label `users/profile`, which is the same context as the +/// identifier the EQL target stores (the same two parts, rendered +/// `users/profile`), so its selectors are the stored document's. An EQL +/// type takes its context as an identifier and runs through `encrypt_as`, +/// not through the chain. +fn plan() -> ValuePlan> { + Plan::value::() + .context("users/profile") + .encrypt_into::() + .build() + .unwrap() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[tokio::test] +async fn a_document_round_trips_through_the_typed_target() { + let (cipher, calls) = common::cipher().await; + let stored = seal(&cipher, &fixture()).await; + assert_eq!(stored.sv.len(), 10, "one entry per node"); + assert_eq!( + calls.lock().unwrap().generate, + vec![vec!["users/profile".to_string()]], + "one data key for the document, under the column's descriptor" + ); + + let wire = serde_json::to_value(&stored).unwrap(); + assert_eq!(wire["v"], 4); + assert_eq!(wire["i"], json!({"t": "users", "c": "profile"})); + assert!(wire["h"].is_string()); + let keys: Vec<&str> = wire + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(keys, ["h", "i", "sv", "v"]); + + let opened: Value = cipher + .decrypt_as(stored.clone(), Default::default()) + .await + .unwrap(); + assert_eq!(opened, fixture()); + let opened: Value = cipher + .decrypt_as(stored.clone(), column().into()) + .await + .unwrap(); + assert_eq!(opened, fixture()); + + let wrong = Identifier::for_column("users", "bio").unwrap(); + let refused = cipher + .decrypt_as::(stored.clone(), wrong.into()) + .await; + assert!( + matches!(refused, Err(stack_encrypt::Error::ContextMismatch { .. })), + "{refused:?}" + ); + assert_eq!( + calls.lock().unwrap().retrieve.len(), + 2, + "the refused open retrieved no key" + ); + + // An entry lifted from the stored document, with the header and its + // ordinal grafted as `->` grafts them, opens to its scalar. + let mut entry = stored.sv[1].clone(); + entry.h = Some(stored.h.clone()); + entry.j = Some(1); + entry.i = Some(stored.i.clone()); + entry.v = Some(stored.v); + let scalar: Value = cipher.decrypt_as(entry, column()).await.unwrap(); + assert_eq!(scalar, json!("world-1")); + let bare = stored.sv[1].clone(); + assert!(cipher.decrypt_as::(bare, column()).await.is_err()); +} + +#[tokio::test] +async fn the_query_operands_are_the_engines_terms_in_the_sql_contract() { + let (cipher, _) = common::cipher().await; + let plan = plan(); + let query = cipher + .query(op::Contains, &json!({"nested": {"list": [{"x": 2}]}})) + .on(&plan) + .await + .unwrap(); + let operand = JsonTreeQuery::try_from(&query).unwrap(); + assert_eq!( + serde_json::to_value(&operand).unwrap(), + serde_json::to_value(&query).unwrap() + ); + assert_eq!(operand.el.len(), 1); + + let bound = cipher + .query(op::AtPath(op::Gte), &("$.number", 1)) + .on(&plan) + .await + .unwrap(); + let leaf = JsonTreeLeafQuery::try_from(&bound).unwrap(); + assert_eq!( + serde_json::to_value(&leaf).unwrap(), + serde_json::to_value(&bound).unwrap() + ); + assert_eq!(leaf.s.0, bound.selector().to_string()); + + // The engine's document under the label and the EQL document under the + // identifier carry the same selectors: one context, two spellings. + let engine = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let typed = seal(&cipher, &fixture()).await; + assert_eq!(engine.entries().len(), typed.sv.len()); + for (a, b) in engine.entries().iter().zip(&typed.sv) { + assert_eq!(hex(a.selector()), b.s.0); + assert_eq!(hex(a.value_selector()), b.vs.0); + } + let opened: Value = cipher.open(engine).using(&plan).await.unwrap(); + assert_eq!(opened, fixture()); +} + +// ============================================================================= +// PostgreSQL +// ============================================================================= + +/// Settle a chain or a pending on the test's runtime: both are +/// `IntoFuture`, not `Future`. +fn wait(runtime: &tokio::runtime::Runtime, f: F) -> F::Output { + runtime.block_on(f.into_future()) +} + +fn connect() -> postgres::Client { + postgres::Client::connect( + &std::env::var("EQL_TEST_DATABASE_URL").expect("EQL_TEST_DATABASE_URL"), + postgres::NoTls, + ) + .unwrap() +} + +/// A small deterministic generator over a few keys and values, so needles +/// drawn from it overlap stored documents often. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + self.0 >> 33 + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + fn value(&mut self, depth: u32) -> Value { + match self.below(if depth == 0 { 4 } else { 6 }) { + 0 => json!(self.below(3)), + 1 => json!(["x", "y", "z"][self.below(3) as usize]), + 2 => json!(self.below(2) == 0), + 3 => Value::Null, + 4 => Value::Array((0..self.below(4)).map(|_| self.value(depth - 1)).collect()), + _ => { + let mut map = Map::new(); + for _ in 0..self.below(4) { + let key = ["a", "b", "c"][self.below(3) as usize].to_owned(); + let _ = map.insert(key, self.value(depth - 1)); + } + Value::Object(map) + } + } + } +} + +/// The design's corpus (`containment-oracle.sql`), the cross-element cases +/// included, plus random documents. +fn corpus() -> Vec { + let mut docs: Vec = [ + r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, + r#"{"a":[{"x":1,"y":2,"z":3}]}"#, + r#"{"a":[{"x":1}]}"#, + r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, + r#"{"a":[{"b":{"c":1,"d":2}},{"b":{"c":3,"d":2}}]}"#, + r#"{"a":[{"b":{"c":1,"d":5}},{"b":{"c":3,"d":2}}]}"#, + r#"{"a":[[1,2],[3,4]]}"#, + r#"{"a":[1,2]}"#, + r#"{"a":{"b":1}}"#, + r#"{"a":{}}"#, + r#"{"a":[1]}"#, + r#"{"a":1}"#, + r#"[1,[2]]"#, + r#"[1,2]"#, + r#"[1]"#, + r#"{"a":null}"#, + r#"{"a":"2"}"#, + r#"{"a":2}"#, + r#"["a"]"#, + r#""a""#, + r#"[["a"]]"#, + r#"{"a":[{"t":[{"u":1,"v":2}]},{"t":[{"u":1},{"v":2}]}]}"#, + r#"{"a":[{"t":[{"u":1},{"v":2}]}]}"#, + r#"{"a":[{}]}"#, + r#"{"a":[[]]}"#, + r#"{}"#, + r#"[]"#, + r#"2"#, + r#"true"#, + r#"null"#, + ] + .iter() + .map(|text| serde_json::from_str(text).unwrap()) + .collect(); + let mut rng = Lcg(7); + for _ in 0..40 { + docs.push(rng.value(3)); + } + docs +} + +/// The needles: the corpus's own documents, plus the design's needles that +/// are not themselves stored. +fn needles() -> Vec { + let mut needles = corpus(); + needles.extend( + [ + r#"{"a":[{"x":1,"y":2}]}"#, + r#"{"a":[{"x":1},{"x":1}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}],"c":4}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3}]}"#, + r#"{"a":[{"b":{"c":1,"d":2}}]}"#, + r#"{"a":[[1,4]]}"#, + r#"{"a":[[1,2]]}"#, + r#"{"a":[[1],[4]]}"#, + r#"{"a":[]}"#, + r#"[2]"#, + r#"[[2]]"#, + r#"[1,1,1]"#, + r#"{"a":2.0}"#, + r#"2.0"#, + r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, + r#"{"a":["x"]}"#, + ] + .iter() + .map(|text| serde_json::from_str::(text).unwrap()), + ); + let mut rng = Lcg(11); + for _ in 0..40 { + needles.push(rng.value(2)); + } + needles +} + +/// Needs an EQL v4 installation. Every document of the corpus is stored +/// encrypted beside its plaintext; for every needle, the rows `@>` and `<@` +/// select over the encrypted column are the rows PostgreSQL's own `jsonb` +/// operators select over the plaintext column. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let plan = plan(); + let mut db = connect(); + db.batch_execute( + "CREATE TEMP TABLE json_tree_docs (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL); + CREATE INDEX json_tree_docs_doc ON json_tree_docs USING gin ((eql_v4.json_tree_query(doc)::jsonb) jsonb_path_ops);", + ) + .unwrap(); + + let docs = corpus(); + for (id, plain) in docs.iter().enumerate() { + let stored = wait(&runtime, seal(&cipher, plain)); + let stored = serde_json::to_string(&stored).unwrap(); + db.execute( + "INSERT INTO json_tree_docs VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_tree)", + &[&(id as i32), &plain.to_string(), &stored], + ) + .unwrap(); + } + + let ids = |db: &mut postgres::Client, sql: &str, param: &str| -> Vec { + db.query(sql, &[¶m]) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect() + }; + let mut checked = 0; + for needle in needles() { + let query = wait(&runtime, cipher.query(op::Contains, &needle).on(&plan)).unwrap(); + let operand = serde_json::to_string(&JsonTreeQuery::try_from(&query).unwrap()).unwrap(); + let plain = needle.to_string(); + let expected = ids( + &mut db, + "SELECT id FROM json_tree_docs WHERE plain @> $1::text::jsonb ORDER BY id", + &plain, + ); + let encrypted = ids( + &mut db, + "SELECT id FROM json_tree_docs WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree ORDER BY id", + &operand, + ); + assert_eq!(encrypted, expected, "@> {needle}"); + let function_form = ids( + &mut db, + "SELECT id FROM json_tree_docs WHERE eql_v4.json_tree_contains(doc, $1::text::jsonb::eql_v4.query_json_tree) ORDER BY id", + &operand, + ); + assert_eq!(function_form, expected, "json_tree_contains {needle}"); + + // `<@`: a stored root scalar inside a needle root array is the one + // case the SQL cannot answer (a stored document carries no `alt`). + let reverse = wait(&runtime, cipher.query(op::ContainedBy, &needle).on(&plan)).unwrap(); + let reverse = serde_json::to_string(&JsonTreeQuery::try_from(&reverse).unwrap()).unwrap(); + let mut expected = ids( + &mut db, + "SELECT id FROM json_tree_docs WHERE plain <@ $1::text::jsonb ORDER BY id", + &plain, + ); + if needle.is_array() { + expected.retain(|id| { + let stored = &docs[*id as usize]; + stored.is_object() || stored.is_array() + }); + } + let encrypted = ids( + &mut db, + "SELECT id FROM json_tree_docs WHERE doc <@ $1::text::jsonb::eql_v4.query_json_tree ORDER BY id", + &reverse, + ); + assert_eq!(encrypted, expected, "<@ {needle}"); + checked += 1; + } + assert!( + checked > 80, + "the corpus should hold many needles: {checked}" + ); + + // The GIN index serves `@>`: with sequential scans discouraged, the + // plan is a bitmap scan over the index. + db.batch_execute("SET enable_seqscan = off").unwrap(); + let needle = wait( + &runtime, + cipher + .query(op::Contains, &json!({"a": [{"x": 1}]})) + .on(&plan), + ) + .unwrap(); + let operand = serde_json::to_string(&JsonTreeQuery::try_from(&needle).unwrap()).unwrap(); + let explain: Vec = db + .query( + "EXPLAIN SELECT id FROM json_tree_docs WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree", + &[&operand], + ) + .unwrap() + .iter() + .map(|r| r.get::<_, String>(0)) + .collect(); + let text = explain.join("\n"); + assert!( + text.contains("Bitmap Index Scan on json_tree_docs_doc"), + "the GIN index should serve containment:\n{text}" + ); + db.batch_execute("SET enable_seqscan = on").unwrap(); + + // Not STRICT, so inlinable; a NULL on either side is still NULL. + let nulls = db + .query_one( + "SELECT (NULL::public.eql_v4_json_tree @> $1::text::jsonb::eql_v4.query_json_tree) IS NULL + AND ((SELECT doc FROM json_tree_docs WHERE id = 0) @> NULL::eql_v4.query_json_tree) IS NULL", + &[&operand], + ) + .unwrap(); + assert!(nulls.get::<_, bool>(0)); +} + +/// Needs an EQL v4 installation. Extraction by selector opens to the leaf; +/// the subtree walk by a client-derived token returns exactly the nodes +/// under it and reassembles; the array functions count and list elements; +/// the orderings at a path select the rows the plaintexts say. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_extracts_walks_and_orders_by_path() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = plan(); + let mut db = connect(); + db.batch_execute( + "CREATE TEMP TABLE json_tree_rows (id int PRIMARY KEY, doc public.eql_v4_json_tree NOT NULL)", + ) + .unwrap(); + let rows = [ + (1, fixture()), + ( + 2, + json!({"hello": "world-2", "nested": {"deep": "other", "list": []}, "number": 7}), + ), + (3, json!({"hello": "aaa", "number": -3, "tags": ["x", "y"]})), + ]; + for (id, plain) in &rows { + let stored = wait(&runtime, seal(&cipher, plain)); + let stored = serde_json::to_string(&stored).unwrap(); + db.execute( + "INSERT INTO json_tree_rows VALUES ($1, $2::text::jsonb::public.eql_v4_json_tree)", + &[id, &stored], + ) + .unwrap(); + } + let selector = |path: &str| { + wait( + &runtime, + keyset.json_selector(&JsonPath::parse(path).unwrap(), column()), + ) + .unwrap() + .to_string() + }; + let subtree_key = |path: &str| { + wait( + &runtime, + keyset.json_subtree_key( + &JsonPath::parse(path).unwrap(), + JsonOptions::default(), + column(), + ), + ) + .unwrap() + }; + let entry = |row: &postgres::Row| -> JsonTreeEntry { + serde_json::from_str(&row.get::<_, String>(0)).unwrap() + }; + let open = + |e: JsonTreeEntry| -> Value { wait(&runtime, cipher.decrypt_as(e, column())).unwrap() }; + + // `->` by selector opens to the leaf; a missing path is NULL. + let hello = selector("$.hello"); + let row = db + .query_one( + "SELECT (doc -> $1::text)::jsonb::text FROM json_tree_rows WHERE id = 2", + &[&hello], + ) + .unwrap(); + assert_eq!(open(entry(&row)), json!("world-2")); + let missing = db + .query_one( + "SELECT (doc -> $1::text)::jsonb::text FROM json_tree_rows WHERE id = 3", + &[&selector("$.nested.deep")], + ) + .unwrap(); + assert!(missing.get::<_, Option>(0).is_none()); + let exists: Vec = db + .query( + "SELECT id FROM json_tree_rows WHERE eql_v4.json_tree_path_exists(doc::jsonb, $1::text) ORDER BY id", + &[&selector("$.nested.deep")], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!(exists, [1, 2]); + + // Every element of an array shares its selector: the path query returns + // one row per element, each opening to its value. + let tags: Vec = db + .query( + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_path_query(doc::jsonb, $1::text) e WHERE id = 3", + &[&selector("$.tags[*]")], + ) + .unwrap() + .iter() + .map(|r| open(entry(r))) + .collect(); + assert_eq!(tags, vec![json!("x"), json!("y")]); + + // The subtree under `$.nested`, from a token derived on the client from + // the path alone: six entries in row 1, reassembling the sub-document; + // two in row 2. + let nested = subtree_key("$.nested"); + for (id, expected) in [ + (1, fixture()["nested"].clone()), + (2, rows[1].1["nested"].clone()), + ] { + let walked: Vec = db + .query( + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text) e WHERE id = $2", + &[&nested.token(), &id], + ) + .unwrap() + .iter() + .map(entry) + .collect(); + assert_eq!(walked.len(), if id == 1 { 6 } else { 2 }); + assert!(walked + .iter() + .all(|e| e.d.is_some() && e.j.is_some() && e.h.is_some())); + let value: Value = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree(walked), column()), + ) + .unwrap(); + assert_eq!(value, expected, "row {id}"); + } + let garbage: i64 = db + .query_one( + "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text) e", + &[&"00".repeat(16)], + ) + .unwrap() + .get(0); + assert_eq!(garbage, 0, "a wrong token unlocks nothing in any row"); + + // A wildcard below the anchor: every element's `x`, each a complete + // subtree, from the list's token gated on the whole path's selector. + let xs = subtree_key("$.nested.list[*].x"); + let walked: Vec = db + .query( + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text) e WHERE id = 1", + &[&xs.token(), &xs.selector().unwrap().to_string()], + ) + .unwrap() + .iter() + .map(entry) + .collect(); + let values: Vec = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree(walked), column()), + ) + .unwrap(); + assert_eq!(values, vec![json!(1), json!(2)]); + + // The array functions, by the list's token. + let list = subtree_key("$.nested.list"); + let counts: Vec<(i32, i32)> = db + .query( + "SELECT id, eql_v4.json_tree_child_count(doc, $1::text) FROM json_tree_rows ORDER BY id", + &[&list.token()], + ) + .unwrap() + .iter() + .map(|r| (r.get(0), r.get(1))) + .collect(); + assert_eq!(counts, [(1, 2), (2, 0), (3, 0)]); + let elements: Vec = db + .query( + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_children(doc, $1::text) e WHERE id = 1", + &[&list.token()], + ) + .unwrap() + .iter() + .map(entry) + .collect(); + assert_eq!(elements.len(), 2); + assert_eq!( + elements.iter().map(|e| e.o).collect::>(), + [Some(0), Some(1)] + ); + + // Orderings at a path: the rows whose `$.number` is in range, and + // sorted by the leaf's term. + let at_least_one = wait( + &runtime, + cipher + .query(op::AtPath(op::Gte), &("$.number", 1)) + .on(&plan), + ) + .unwrap(); + let at_least_one = + serde_json::to_string(&JsonTreeLeafQuery::try_from(&at_least_one).unwrap()).unwrap(); + let number = selector("$.number"); + let ids: Vec = db + .query( + "SELECT id FROM json_tree_rows WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf ORDER BY id", + &[&number, &at_least_one], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!(ids, [1, 2]); + let ids: Vec = db + .query( + "SELECT id FROM json_tree_rows WHERE (doc -> $1::text) < $2::text::jsonb::eql_v4.query_json_tree_leaf ORDER BY id", + &[&number, &at_least_one], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!(ids, [3]); + let sorted: Vec = db + .query( + "SELECT id FROM json_tree_rows ORDER BY eql_v4.ord_term(doc -> $1::text), id", + &[&number], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!(sorted, [3, 1, 2], "-3, 1, 7"); + let text = selector("$.hello"); + let sorted: Vec = db + .query( + "SELECT id FROM json_tree_rows ORDER BY eql_v4.ord_term(doc -> $1::text), id", + &[&text], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!(sorted, [3, 1, 2], "aaa, world-1, world-2"); + + // An ordering operand that lost its `query_json_tree_leaf` cast is plain + // jsonb, and every comparison of an entry with plain jsonb raises, + // either way round, instead of comparing the entry's ciphertext. + for op in ["<", "<=", ">", ">=", "=", "<>"] { + for sql in [ + format!("SELECT id FROM json_tree_rows WHERE (doc -> $1::text) {op} $2::text::jsonb"), + format!("SELECT id FROM json_tree_rows WHERE $2::text::jsonb {op} (doc -> $1::text)"), + ] { + match db.query(&sql, &[&number, &at_least_one]) { + Ok(rows) => panic!("{sql} must be blocked, but returned {} rows", rows.len()), + Err(error) => { + let message = error + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_else(|| error.to_string()); + assert!( + message.contains(&format!( + "operator {op} is not supported for public.eql_v4_json_tree_entry" + )), + "{sql}: {message}" + ); + } + } + } + } + + // The native jsonb operators are blocked on the column. The operand is + // typed, as every EQL jsonb domain requires: an untyped literal resolves + // to the native operator on the base type. + match db.query("SELECT doc ? 'hello'::text FROM json_tree_rows", &[]) { + Ok(rows) => panic!( + "native `?` must be blocked, but returned {} rows", + rows.len() + ), + Err(error) => { + let message = error + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_else(|| error.to_string()); + assert!( + message.contains("operator ? is not supported for public.eql_v4_json_tree"), + "native `?` must be blocked: {message}" + ); + } + } +} diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index ad3020dba..523ea850e 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -184,10 +184,16 @@ mod given_text_eq { async fn query_bytes_are_the_typed_query_twin() { let (cipher, calls) = common::cipher().await; let keyset = cipher.default_keyset(); - let by_name = targets::query("TextEq", &keyset, &email(), text("alice@example.com")) - .unwrap() - .await - .unwrap(); + let by_name = targets::query( + "TextEq", + &common::index_of("TextEq"), + &keyset, + &email(), + text("alice@example.com"), + ) + .unwrap() + .await + .unwrap(); let typed: TextEqQuery = keyset .encrypt_as(&"alice@example.com".to_owned(), column()) .await @@ -255,6 +261,7 @@ mod given_text_eq { ); let error = refused(targets::query( "TextEq", + &common::index_of("TextEq"), &keyset, &email(), frame(None::), @@ -354,7 +361,13 @@ mod given_a_name_the_engine_cannot_produce { name, ); unproducible( - refused(targets::query(name, &keyset, &email(), text("x"))), + refused(targets::query( + name, + &common::index_of(name), + &keyset, + &email(), + text("x"), + )), name, ); unproducible( @@ -362,15 +375,15 @@ mod given_a_name_the_engine_cannot_produce { name, ); } - let error = refused(targets::encrypt("TextOrdOre", &keyset, &email(), text("x"))); + let error = refused(targets::encrypt("DateOrdOre", &keyset, &email(), text("x"))); assert_eq!( error.to_string(), - "the engine cannot produce TextOrdOre yet: EQL stores block ORE terms and the engine \ - derives CLLW ORE (#1132)" + "the engine cannot produce DateOrdOre yet: the engine derives no order term for a date, \ + timestamp or decimal yet: how they are ordered is not decided (#1236)" ); // Refused by name before the label or the value is looked at. let error = refused(targets::encrypt( - "TextOrdOre", + "DateOrdOre", &keyset, &Label::new(["one"]).unwrap(), frame(None::), @@ -404,7 +417,13 @@ mod given_an_unknown_name { &email(), text("x"), ))); - unknown(refused(targets::query(name, &keyset, &email(), text("x")))); + unknown(refused(targets::query( + name, + &common::index_of(name), + &keyset, + &email(), + text("x"), + ))); unknown(refused(targets::decrypt(name, &cipher, &email(), b"{}"))); } assert_eq!( @@ -434,7 +453,7 @@ mod the_table { match term { Term::Hm => "eq", Term::Bloom => "match", - Term::Ore => "ore", + Term::Ore => "ore_block", Term::Ope => "ope", } } @@ -565,10 +584,10 @@ mod the_table { /// The types the engine produces in EQL v4 today: the storage and /// equality domains of every family whose plaintext eql-bindings names, - /// the OPE-ordered domains of those the engine derives an order term - /// for, and text match. + /// the ordered domains (OPE and block ORE) of those the engine derives + /// an order term for, and text match and search. #[test] - fn the_producible_types_are_the_ones_with_no_ore_and_a_rust_plaintext() { + fn the_producible_types_are_the_ones_with_a_rust_plaintext() { let producible: Vec<&str> = targets::targets() .iter() .filter(|t| t.producible) @@ -579,14 +598,17 @@ mod the_table { [ "Integer", "IntegerEq", + "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", "Smallint", "SmallintEq", + "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", "Bigint", "BigintEq", + "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", @@ -598,15 +620,20 @@ mod the_table { "Text", "TextEq", "TextMatch", + "TextOrdOre", "TextOrd", "TextOrdOpe", + "TextSearchOre", + "TextSearch", "Boolean", "Real", "RealEq", + "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", + "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe" ], @@ -619,8 +646,6 @@ mod the_table { ("date" | "timestamp" | "numeric", "Ord" | "OrdOpe" | "OrdOre") => { "no order term for a date, timestamp or decimal yet" } - (_, "OrdOre" | "SearchOre") => "derives CLLW ORE (#1132)", - ("text", "Search") => "carries a match term", (family, suffix) => panic!("{}: unexpected {family} {suffix}", row.name), }; assert!(reason.contains(expected), "{}: {reason}", row.name); @@ -687,10 +712,16 @@ mod the_cross_language_fixture { async fn the_text_eq_query_fixture_is_what_the_dispatch_derives() { let (cipher, calls) = common::cipher().await; let keyset = cipher.default_keyset(); - let query = targets::query("TextEq", &keyset, &email(), text(PLAINTEXT)) - .unwrap() - .await - .unwrap(); + let query = targets::query( + "TextEq", + &common::index_of("TextEq"), + &keyset, + &email(), + text(PLAINTEXT), + ) + .unwrap() + .await + .unwrap(); assert!( calls.lock().unwrap().generate.is_empty(), "a query mints nothing" diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs new file mode 100644 index 000000000..1f5ec021d --- /dev/null +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -0,0 +1,542 @@ +//! Rust queries an EQL v4 field through Stack Encrypt's typed query chain +//! (ADR-0009): `cipher.query(op::Eq, v).on(&plan)` on a plan for the type, +//! answered through the `op::Answers` impls eql-codegen writes, with the +//! type's query twin as the term. +//! +//! The typed path must agree with the data path a binding takes +//! (`stack_encrypt::dynamic::query_term`, which the Go SDK runs) on three +//! things: which operators each type answers, the bytes of each query, and +//! for `TextMatch`, how a decrypted row verifies and how many rows to fetch. + +mod common; + +use eql_bindings::encryption::query::{MatchTwin, QueryLabelError}; +use eql_bindings::encryption::targets::{self as eql_targets, Target}; +use eql_bindings::v4::answers::ANSWERS; +use eql_bindings::v4::plaintext::{Date, Decimal, Timestamp}; +use eql_bindings::v4::text::{TextEq, TextMatch, TextMatchQuery}; +use eql_bindings::v4::{ + bigint, date, double, integer, numeric, real, smallint, timestamp, DomainType, +}; +use eql_bindings::Identifier; +use serde::Serialize; +use stack_encrypt::dynamic::{ + self, record, FieldPlan, OpSpec, Operand, TargetDescriptor, TargetError, TargetResolver, + ValueFrame, +}; +use stack_encrypt::op::{self, Probabilistic, Stats}; +use stack_encrypt::{ + ContextPiece, Encrypt, Error, KeysetCipher, Label, NonEmpty, Pending, Plan, StackCipher, +}; + +type Cipher = StackCipher; + +/// A Rust value as the engine's runtime plaintext: its frame. +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() +} + +/// eql-bindings' table as the engine's resolver, as the Go guest installs +/// it (`languages/golang/encrypt/guest/src/targets.rs`). +struct Eql; + +/// A table row as the engine's descriptor, with the match options of a type +/// that carries a match index, as the guest describes it. +fn describe(target: &Target) -> TargetDescriptor { + let descriptor = TargetDescriptor::new( + target.name, + target.family, + target.suffix, + target.plaintext_kind(), + target.sql_domain, + target.indexes.iter().map(|key| (*key).to_owned()).collect(), + target.query.map(str::to_owned), + target.query_sql_domain.map(str::to_owned), + target.producible, + target.reason.map(str::to_owned), + ); + match target.match_options() { + Some(options) => descriptor.with_match_options(options), + None => descriptor, + } +} + +fn convert(error: eql_targets::TargetError) -> TargetError { + TargetError::Other(Box::new(error)) +} + +impl TargetResolver for Eql { + fn targets(&self) -> Vec { + eql_targets::targets().iter().map(describe).collect() + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::encrypt(name, keyset, label, plaintext).map_err(convert) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + stored: &[u8], + ) -> Result, TargetError> { + eql_targets::decrypt(name, cipher, label, stored).map_err(convert) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &stack_encrypt::target::IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::query(name, index, keyset, label, plaintext).map_err(convert) + } +} + +/// A data plan with one field, `value`, under `users/`, naming the +/// EQL type `name`. +fn data_plan(name: &str, column: &str) -> record::Plan { + let context = NonEmpty::new(ContextPiece::List(vec![ + ContextPiece::Text("users".into()), + ContextPiece::Text(column.to_owned().into()), + ])) + .unwrap(); + let field = FieldPlan::with_target("value", context, name).unwrap(); + record::Plan::new_with(vec![field], &Eql).unwrap() +} + +/// The data path's term for the query, and the domain it is cast to. +async fn data_term( + cipher: &Cipher, + name: &str, + column: &str, + operator: OpSpec, + input: ValueFrame, +) -> (Vec, String) { + let keyset = cipher.default_keyset(); + let term = dynamic::query_term( + &keyset, + &data_plan(name, column), + "value", + operator, + input, + &Eql, + ) + .unwrap() + .await + .unwrap(); + let Operand::One(bytes) = term.operand() else { + panic!("{name}: a one-value query has one operand"); + }; + (bytes.clone(), term.domain().unwrap().to_owned()) +} + +/// The bytes a typed query term binds, and the query domain it names. +fn typed_bytes(query: &Q) -> (Vec, String) { + ( + serde_json::to_vec(query).unwrap(), + Q::sql_domain_static().to_owned(), + ) +} + +/// A typed plan for the EQL type `$ty` over `$plaintext`, under +/// `users/`, and the term of `cipher.query($op, $value)` on it. +macro_rules! typed { + ($cipher:expr, $column:expr, $ty:ty, $plaintext:ty, $op:expr, $value:expr) => {{ + let plan = Plan::value::<$plaintext>() + .context(format!("users/{}", $column)) + .encrypt_into::<$ty>() + .build() + .unwrap(); + $cipher.query($op, $value).on(&plan).await.unwrap() + }}; +} + +/// The typed query for each operator a type answers, with the input the +/// data path is given for the same query: one arm per row of `ANSWERS`, so +/// a type that starts answering an operator fails here until it has one, +/// and an arm compiles only if the type implements `op::Answers` for it. +async fn typed_and_input( + cipher: &Cipher, + name: &str, + operator: &str, + column: &str, +) -> ((Vec, String), ValueFrame) { + match (name, operator) { + ("TextEq", "eq") => ( + typed_bytes(&typed!( + cipher, + column, + TextEq, + String, + op::Eq, + "bob@example.com" + )), + frame("bob@example.com"), + ), + ("IntegerEq", "eq") => ( + typed_bytes(&typed!( + cipher, + column, + integer::IntegerEq, + i32, + op::Eq, + &-42i32 + )), + frame(-42i32), + ), + ("SmallintEq", "eq") => ( + typed_bytes(&typed!( + cipher, + column, + smallint::SmallintEq, + i16, + op::Eq, + &-7i16 + )), + frame(-7i16), + ), + ("DateEq", "eq") => { + let day = || Date::from_days_since_epoch(19_000).expect("a day in range"); + ( + typed_bytes(&typed!(cipher, column, date::DateEq, Date, op::Eq, &day())), + frame(day()), + ) + } + ("TimestampEq", "eq") => { + // Sub-microsecond nanoseconds: both paths truncate them alike. + let instant = + || Timestamp::new(1_700_000_000, 123_456_789).expect("an instant in range"); + ( + typed_bytes(&typed!( + cipher, + column, + timestamp::TimestampEq, + Timestamp, + op::Eq, + &instant() + )), + frame(instant()), + ) + } + ("NumericEq", "eq") => { + let amount = || Decimal::new(-1_234_567, 4).expect("a decimal in range"); + ( + typed_bytes(&typed!( + cipher, + column, + numeric::NumericEq, + Decimal, + op::Eq, + &amount() + )), + frame(amount()), + ) + } + ("BigintEq", "eq") => ( + typed_bytes(&typed!( + cipher, + column, + bigint::BigintEq, + i64, + op::Eq, + &(i64::MIN + 1) + )), + frame(i64::MIN + 1), + ), + ("RealEq", "eq") => ( + typed_bytes(&typed!(cipher, column, real::RealEq, f32, op::Eq, &-0.0f32)), + frame(-0.0f32), + ), + ("DoubleEq", "eq") => ( + typed_bytes(&typed!( + cipher, + column, + double::DoubleEq, + f64, + op::Eq, + &f64::NAN + )), + frame(f64::NAN), + ), + ("TextMatch", "matches") => { + let query: MatchTwin = + typed!(cipher, column, TextMatch, String, op::Matches, "alice"); + (typed_bytes(query.twin()), frame("alice")) + } + _ => panic!( + "{name} answers `{operator}` in the typed query chain: give it an arm here, with the \ + input the data path is given" + ), + } +} + +/// The operators each type answers in the typed chain are exactly the ones +/// the data path answers for a field that names it, type by type, in order, +/// for every type whose twin the chain assembles: one term, from an equality +/// or a match index. A producible type whose twin needs an order term, or +/// several terms, is the chain's one gap: the data path answers it and the +/// typed chain answers nothing, and this holds the gap to exactly those +/// types, so a type with an equality or match twin cannot drop out unseen. +/// The typed impls and `ANSWERS` are generated from one rule, so this is +/// what holds that rule to the engine's. +#[test] +fn the_typed_chain_answers_what_the_data_path_answers() { + let mut gap = Vec::new(); + for target in eql_targets::targets() { + let data: Vec<&str> = describe(target) + .operators() + .into_iter() + .map(OpSpec::name) + .collect(); + let typed: Vec<&str> = ANSWERS + .iter() + .find(|(name, _)| *name == target.name) + .map(|(_, ops)| ops.to_vec()) + .unwrap_or_default(); + let one_term = target.indexes.len() <= 1 + && target + .indexes + .iter() + .all(|index| matches!(*index, "eq" | "match")); + if one_term { + assert_eq!(typed, data, "{}", target.name); + } else { + assert!( + typed.is_empty(), + "{}: the typed chain answers nothing yet", + target.name + ); + if !data.is_empty() { + gap.push(target.name); + } + } + } + // The gap today: every producible ordered type (OPE and block ORE), and + // text search, whose twins carry an order term among their three. A type + // leaves this list + // when the typed chain assembles its twin, and its answers are then + // held to the data path's above. + assert_eq!( + gap, + [ + "IntegerOrdOre", + "IntegerOrd", + "IntegerOrdOpe", + "SmallintOrdOre", + "SmallintOrd", + "SmallintOrdOpe", + "BigintOrdOre", + "BigintOrd", + "BigintOrdOpe", + "TextOrdOre", + "TextOrd", + "TextOrdOpe", + "TextSearchOre", + "TextSearch", + "RealOrdOre", + "RealOrd", + "RealOrdOpe", + "DoubleOrdOre", + "DoubleOrd", + "DoubleOrdOpe", + ] + ); + for (name, _) in ANSWERS { + assert!(eql_targets::target(name).is_some(), "{name} is a target"); + } + assert!(!ANSWERS.is_empty()); +} + +/// Every typed query binds the bytes the data path derives for the same +/// field, operator and value, under the same query domain. The float +/// values are the ones whose terms the semantic baseline folds (`-0.0`, +/// NaN). +#[tokio::test] +async fn every_typed_query_is_the_data_paths_query() { + let (cipher, _) = common::cipher().await; + for (name, operators) in ANSWERS { + for operator in *operators { + let column = "col"; + let (typed, input) = typed_and_input(&cipher, name, operator, column).await; + let op = OpSpec::parse(operator).unwrap(); + let data = data_term(&cipher, name, column, op, input).await; + assert_eq!(typed, data, "{name} {operator}"); + let json: serde_json::Value = serde_json::from_slice(&typed.0).unwrap(); + assert_eq!(json["v"], 4, "{name}"); + assert_eq!(json["i"], serde_json::json!({"t": "users", "c": column})); + } + } +} + +/// The typed `TextEq` query is the stored value's term under the same +/// column, as the data path's is. +#[tokio::test] +async fn a_typed_eq_query_matches_the_stored_term() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let column = Identifier::for_column("users", "email").unwrap(); + let stored: TextEq = keyset + .encrypt_as(&"bob@example.com".to_owned(), column) + .await + .unwrap(); + let query = typed!(cipher, "email", TextEq, String, op::Eq, "bob@example.com"); + assert_eq!(query.hm, stored.hm); + assert_eq!(query.i, stored.i); + let other = typed!(cipher, "email", TextEq, String, op::Eq, "alice@example.com"); + assert_ne!(other.hm, stored.hm); +} + +/// `verify` and `target_limit` on the typed `TextMatch` query answer as +/// the data path's `verify` and `target_limit` do for the same query. +#[tokio::test] +async fn a_typed_match_query_verifies_and_sizes_as_the_data_path_does() { + let (cipher, _) = common::cipher().await; + let plan = data_plan("TextMatch", "name"); + for needle in ["alice", "ali", "ALICE SMITH"] { + let query: MatchTwin = + typed!(cipher, "name", TextMatch, String, op::Matches, needle); + let input = frame(needle); + + let page = ["Alice Smith", "alicia", "Bob", "malice", "ALICE", "al", ""]; + let data = + dynamic::verify(&plan, "value", OpSpec::Matches, input.clone(), page, &Eql).unwrap(); + let typed: Vec = page.iter().map(|v| query.verify(v)).collect(); + assert_eq!(typed, data, "{needle}"); + assert!(query.verify("Alice Smith"), "{needle}"); + assert!(!query.verify("Bob"), "{needle}"); + + let (term, _) = data_term(&cipher, "TextMatch", "name", OpSpec::Matches, input).await; + assert_eq!( + serde_json::to_vec(&query).unwrap(), + term, + "serializes as the twin" + ); + for stats in [ + Stats::measured(1_000_000, [(40, 500_000), (120, 500_000)]), + Stats::measured(1_000_000, [(40, 1_000_000)]).with_confidence(0.5), + Stats::declared(1_000_000), + ] { + for want in [0, 1, 10, 1_000] { + let data = dynamic::target_limit( + &plan, + "value", + OpSpec::Matches, + &term, + want, + &stats, + &Eql, + ) + .unwrap(); + assert_eq!(query.target_limit(want, &stats), data, "{needle} {want}"); + } + } + } +} + +/// The query's refusals are the data path's: match text that cuts no token +/// is refused before any term exists, and a label that names no column is +/// refused, since the twin stores the column. Both come before the named +/// keyset is loaded. +#[tokio::test] +async fn a_typed_query_refuses_what_the_data_path_refuses() { + let (cipher, calls) = common::cipher().await; + let plan = Plan::value::() + .context("users/name") + .encrypt_into::() + .build() + .unwrap(); + let short = cipher + .query(op::Matches, "al") + .keyset("tenant") + .on(&plan) + .await; + assert!( + matches!( + short, + Err(Error::Term(stack_encrypt::sem::TermError::EmptyTermText)) + ), + "{short:?}" + ); + + for label in ["users", "public/users/email"] { + let plan = Plan::value::() + .context(label) + .encrypt_into::() + .build() + .unwrap(); + let refused = cipher.query(op::Eq, "bob").keyset("tenant").on(&plan).await; + let Err(Error::Other(reason)) = &refused else { + panic!("{label}: {refused:?}"); + }; + assert_eq!( + reason.downcast_ref::(), + Some(&QueryLabelError::NotAColumn { + label: label.to_owned() + }), + "{label}: {reason}" + ); + } + assert_eq!(calls.lock().unwrap().loads, 0, "refused before any keyset"); +} + +/// A query extended with `.extend(..)` derives its terms under the column +/// and the extension, while the twin stores the column alone, so it could +/// match nothing: it is refused, before the named keyset is loaded, for an +/// exact type and for `TextMatch`. +#[tokio::test] +async fn an_extended_typed_query_is_refused_before_any_keyset() { + let (cipher, calls) = common::cipher().await; + let email = Plan::value::() + .context("users/email") + .encrypt_into::() + .build() + .unwrap(); + let name = Plan::value::() + .context("users/name") + .encrypt_into::() + .build() + .unwrap(); + let refusals = [ + cipher + .query(op::Eq, "bob") + .extend(7i64) + .keyset("tenant") + .on(&email) + .await + .map(drop), + cipher + .query(op::Matches, "alice") + .extend(7i64) + .keyset("tenant") + .on(&name) + .await + .map(drop), + ]; + for refused in refusals { + let Err(Error::Other(reason)) = &refused else { + panic!("{refused:?}"); + }; + assert_eq!( + reason.downcast_ref::(), + Some(&QueryLabelError::Extended), + "{reason}" + ); + } + assert_eq!(calls.lock().unwrap().loads, 0, "refused before any keyset"); + + // Unextended, the same query loads the keyset and runs. + cipher + .query(op::Eq, "bob") + .keyset("tenant") + .on(&email) + .await + .unwrap(); + assert_eq!(calls.lock().unwrap().loads, 1); +} diff --git a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs index 8026446eb..466e7377c 100644 --- a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs +++ b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs @@ -91,7 +91,7 @@ async fn hm( name: &str, value: ValueFrame, ) -> String { - let bytes = targets::query(name, keyset, &label(), value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label(), value) .unwrap() .await .unwrap(); @@ -382,7 +382,7 @@ async fn the_ledger_query_fixture_is_what_the_dispatch_derives() { let keyset = &keyset; async move { let label = Label::new(["profiles", column]).unwrap(); - let bytes = targets::query(name, keyset, &label, value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label, value) .unwrap() .await .unwrap(); diff --git a/packages/eql/tests/encryption/tests/v4_order.rs b/packages/eql/tests/encryption/tests/v4_order.rs index f9f76da38..d0567e2d0 100644 --- a/packages/eql/tests/encryption/tests/v4_order.rs +++ b/packages/eql/tests/encryption/tests/v4_order.rs @@ -66,7 +66,7 @@ async fn query( name: &str, value: ValueFrame, ) -> serde_json::Value { - let bytes = targets::query(name, keyset, &label(), value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label(), value) .unwrap() .await .unwrap(); @@ -81,21 +81,23 @@ fn op(value: &serde_json::Value) -> &str { value["op"].as_str().expect("an ordered value carries op") } -/// Every producible ordered type is in [`ORDERED`] and the other way -/// round: a type that becomes producible is covered here or fails. +/// Every producible `Ord` and `OrdOpe` type is in [`ORDERED`] and the other +/// way round: a type that becomes producible is covered here or fails. +/// `TextSearch` carries an OPE term too; its query carries one term per +/// operator, and `v4_text_search.rs` covers it. #[test] fn the_ordered_producible_types_are_these() { let mut ordered: Vec<&str> = targets::targets() .iter() .filter(|t| t.producible && t.indexes.contains(&"ope")) .map(|t| t.name) + .filter(|name| *name != "TextSearch") .collect(); ordered.sort_unstable(); let mut expected = ORDERED.to_vec(); expected.sort_unstable(); assert_eq!(ordered, expected); - // No producible type carries an ORE term: EQL's is block ORE, and the - // engine's is CLLW ORE. + // No type carries a CLLW ORE term: EQL's ORE term is block ORE. assert!(targets::targets() .iter() .filter(|t| t.producible) @@ -340,7 +342,13 @@ async fn text_with_no_fold_is_refused() { Err(error) => panic!("{name}: refused before the plan ran: {error}"), }; assert!(stored.is_err(), "{name}: stored text with no fold"); - let queried = match targets::query(name, &keyset, &label(), frame(unassigned)) { + let queried = match targets::query( + name, + &common::index_of(name), + &keyset, + &label(), + frame(unassigned), + ) { Ok(pending) => pending.await, Err(error) => panic!("{name}: refused before the plan ran: {error}"), }; @@ -406,7 +414,7 @@ async fn the_ordered_query_fixture_is_what_the_dispatch_derives() { let keyset = &keyset; async move { let label = Label::new(["rankings", column]).unwrap(); - let bytes = targets::query(name, keyset, &label, value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label, value) .unwrap() .await .unwrap(); @@ -682,10 +690,16 @@ fn postgres_compares_and_orders_ope_rows() { let probes: Vec = runtime.block_on(async { let mut probes = Vec::new(); for ((column, _, _), (name, value)) in COLUMNS.iter().zip(frames(&rows[1].1)) { - let bytes = targets::query(name, &keyset, &under(column), value) - .unwrap() - .await - .unwrap(); + let bytes = targets::query( + name, + &common::index_of(name), + &keyset, + &under(column), + value, + ) + .unwrap() + .await + .unwrap(); probes.push(String::from_utf8(bytes).unwrap()); } probes diff --git a/packages/eql/tests/encryption/tests/v4_ore_block.rs b/packages/eql/tests/encryption/tests/v4_ore_block.rs new file mode 100644 index 000000000..8cd02b70b --- /dev/null +++ b/packages/eql/tests/encryption/tests/v4_ore_block.rs @@ -0,0 +1,797 @@ +//! The block ORE EQL v4 types Stack Encrypt produces: `IntegerOrdOre`, +//! `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, `DoubleOrdOre`, +//! `TextOrdOre` and `TextSearchOre`, and EQL v4's comparator for their `ob` term. +//! +//! The term is ore-rs's 6-bit block ORE: the fixed-width scheme (wire +//! header `02 02`) for the integers and floats, the chained scheme (`02 03`, +//! text padded to a power-of-two length bucket of at least 16 bytes) for +//! text. Every term carries a random nonce, so the stored bytes differ on +//! each write and only the comparator orders them. +//! +//! Against an EQL v4 database these tests compare pairs of real ore-rs +//! ciphertexts with `eql_v4_internal.compare_ore_block_v2_term`, both ways +//! round, and require the answer to be the plaintext order and ore-rs's own +//! comparator's: equal values, a first difference in every block, shared +//! prefixes, and text in different padding buckets. They check the hash bit +//! against ore.rs's vectors, that malformed and mismatched terms raise, that +//! EQL admits the block counts ore-rs does up to 1366 blocks (the longest +//! term Stack Encrypt writes) and none past it, and that the domains answer +//! `<`, `=` and `ORDER BY` through a btree index. + +mod common; + +use std::cmp::Ordering; + +use eql_bindings::encryption::targets; +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::sem::{Orderable, OreBlockTerm}; +use stack_encrypt::{Encrypt, Label}; + +type Keyset<'a> = stack_encrypt::KeysetCipher<'a, common::Observed>; + +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() +} + +fn label() -> Label { + Label::new(["rankings", "level"]).unwrap() +} + +fn unhex(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// The stored JSON the named type writes for `value`. +async fn stored(keyset: &Keyset<'_>, name: &str, value: ValueFrame) -> serde_json::Value { + let bytes = targets::encrypt(name, keyset, &label(), value) + .unwrap() + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +/// The one block ORE term the named type stores for `value`, as bytes. +async fn ob(keyset: &Keyset<'_>, name: &str, value: ValueFrame) -> Vec { + let json = stored(keyset, name, value).await; + let terms = json["ob"].as_array().expect("ob is an array"); + assert_eq!(terms.len(), 1, "{name}: one block ORE term"); + unhex(terms[0].as_str().unwrap()) +} + +/// The terms of `values`, two writes each: the second write of a value is +/// a different ciphertext of the same plaintext. +async fn terms(keyset: &Keyset<'_>, name: &str, values: &[T]) -> Vec> { + let mut out = Vec::new(); + for value in values { + out.push(ob(keyset, name, frame(*value)).await); + out.push(ob(keyset, name, frame(*value)).await); + } + out +} + +async fn text_terms(keyset: &Keyset<'_>, name: &str, values: &[&str]) -> Vec> { + let mut out = Vec::new(); + for value in values { + out.push(ob(keyset, name, frame(*value)).await); + out.push(ob(keyset, name, frame(*value)).await); + } + out +} + +/// ore-rs's own comparator over two terms. +fn rust_cmp(a: &[u8], b: &[u8]) -> Ordering { + let a = OreBlockTerm::::from_bytes(a).unwrap(); + let b = OreBlockTerm::::from_bytes(b).unwrap(); + a.compare(&b).expect("terms of one index compare") +} + +#[tokio::test] +async fn each_type_stores_one_block_ore_term_of_its_scheme() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + for (name, value, header, keys) in [ + ( + "IntegerOrdOre", + frame(7i32), + [2u8, 2], + &["c", "i", "ob", "v"][..], + ), + ( + "SmallintOrdOre", + frame(7i16), + [2, 2], + &["c", "i", "ob", "v"], + ), + ("BigintOrdOre", frame(7i64), [2, 2], &["c", "i", "ob", "v"]), + ("RealOrdOre", frame(7.0f32), [2, 2], &["c", "i", "ob", "v"]), + ( + "DoubleOrdOre", + frame(7.0f64), + [2, 2], + &["c", "i", "ob", "v"], + ), + ( + "TextOrdOre", + frame("alice"), + [2, 3], + &["c", "hm", "i", "ob", "v"], + ), + ( + "TextSearchOre", + frame("alice"), + [2, 3], + &["bf", "c", "hm", "i", "ob", "v"], + ), + ] { + let json = stored(&keyset, name, value).await; + let mut found: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + found.sort_unstable(); + assert_eq!(found, keys, "{name}"); + let term = unhex(json["ob"][0].as_str().unwrap()); + assert_eq!(term[..2], header, "{name}: the scheme's wire header"); + } +} + +/// Pairs whose order keys first differ in each block in turn: `x` with one +/// bit of block `k` clear, and `x` with it set. An `i32`'s key is 32 bits, +/// six 6-bit blocks (the last holds two bits); an `i64`'s is eleven. +fn first_difference_in_every_block_i32() -> Vec { + let mut values = Vec::new(); + for block in 0..6u32 { + let bit = (6 * block + 5).min(31); + let mask = 1u32 << (31 - bit); + let key = 0x5a5a_5a5a_u32 & !mask; + // The order key of an i32 is its bits with the sign bit flipped. + values.push((key ^ 0x8000_0000) as i32); + values.push(((key | mask) ^ 0x8000_0000) as i32); + } + values +} + +fn first_difference_in_every_block_i64() -> Vec { + let mut values = Vec::new(); + for block in 0..11u32 { + let bit = (6 * block + 5).min(63); + let mask = 1u64 << (63 - bit); + let key = 0x5a5a_5a5a_5a5a_5a5a_u64 & !mask; + values.push((key ^ 0x8000_0000_0000_0000) as i64); + values.push(((key | mask) ^ 0x8000_0000_0000_0000) as i64); + } + values +} + +/// Text sharing prefixes, across padding buckets (a string pads to 16, 32 +/// or 64 bytes, its one marker byte counted), and differing first at each +/// position of a 40-character string, so in every block of it. +fn texts() -> Vec { + let mut values: Vec = [ + "", + "a", + "ab", + "abc", + "abd", + "abc ", + "b", + "zz", + // 15 characters (a 16-byte bucket) and 16 (a 32-byte bucket). + "aaaaaaaaaaaaaaa", + "aaaaaaaaaaaaaaaa", + "aaaaaaaaaaaaaaab", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + let base = "mnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz".to_string(); + for at in 0..base.len() { + let mut changed = base.clone().into_bytes(); + changed[at] = b'a' + (changed[at] - b'a' + 1) % 26; + values.push(String::from_utf8(changed).unwrap()); + } + values.push(base); + values +} + +fn connect() -> postgres::Client { + postgres::Client::connect( + &std::env::var("EQL_TEST_DATABASE_URL").expect("EQL_TEST_DATABASE_URL"), + postgres::NoTls, + ) + .unwrap() +} + +/// EQL v4's comparator over two terms. +fn sql_cmp(db: &mut postgres::Client, a: &[u8], b: &[u8]) -> Result { + db.query_one( + "SELECT eql_v4_internal.compare_ore_block_v2_term(\ + ROW($1::bytea)::eql_v4_internal.ore_block_v2_term, \ + ROW($2::bytea)::eql_v4_internal.ore_block_v2_term)", + &[&a, &b], + ) + .map(|row| row.get(0)) +} + +fn as_ordering(n: i32) -> Ordering { + n.cmp(&0) +} + +/// Every pair of `terms`, both ways round: EQL's answer, ore-rs's and the +/// plaintext order all agree. `terms[i]` encrypts `plain[i / 2]`. +fn assert_agree( + db: &mut postgres::Client, + what: &str, + plain: &[P], + terms: &[Vec], +) { + for (i, a) in terms.iter().enumerate() { + for (j, b) in terms.iter().enumerate() { + let (pa, pb) = (&plain[i / 2], &plain[j / 2]); + let expected = pa.partial_cmp(pb).unwrap(); + let sql = as_ordering(sql_cmp(db, a, b).unwrap_or_else(|e| panic!("{what}: {e}"))); + assert_eq!(sql, expected, "{what}: {pa:?} against {pb:?} in SQL"); + assert_eq!( + rust_cmp::(a, b), + expected, + "{what}: {pa:?} against {pb:?} in ore-rs" + ); + } + } +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_compares_fixed_width_terms_as_ore_rs_and_the_plaintexts_do() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + + let mut ints = vec![i32::MIN, -1_000_000, -1, 0, 1, 63, 64, 1_000_000, i32::MAX]; + ints.extend(first_difference_in_every_block_i32()); + ints.sort_unstable(); + ints.dedup(); + let int_terms = runtime.block_on(terms(&keyset, "IntegerOrdOre", &ints)); + assert_agree::(&mut db, "IntegerOrdOre", &ints, &int_terms); + + let smalls = vec![i16::MIN, -300, -1, 0, 1, 63, 64, 300, i16::MAX]; + let small_terms = runtime.block_on(terms(&keyset, "SmallintOrdOre", &smalls)); + assert_agree::(&mut db, "SmallintOrdOre", &smalls, &small_terms); + + let mut bigs = vec![i64::MIN, -1, 0, 1, 1 << 40, i64::MAX]; + bigs.extend(first_difference_in_every_block_i64()); + bigs.sort_unstable(); + bigs.dedup(); + let big_terms = runtime.block_on(terms(&keyset, "BigintOrdOre", &bigs)); + assert_agree::(&mut db, "BigintOrdOre", &bigs, &big_terms); + + // Floats order as float8: -0.0 ties 0.0, and NaN (left out of the + // plaintext comparison, which has no order for it) sorts last, below. + let doubles = [ + f64::NEG_INFINITY, + -1e300, + -1.5, + -0.0, + 0.0, + 1e-300, + 1.0, + 2.0, + 1e300, + f64::INFINITY, + ]; + let double_terms = runtime.block_on(terms(&keyset, "DoubleOrdOre", &doubles)); + assert_agree::(&mut db, "DoubleOrdOre", &doubles, &double_terms); + let nan = runtime.block_on(ob(&keyset, "DoubleOrdOre", frame(f64::NAN))); + for term in &double_terms { + assert_eq!(sql_cmp(&mut db, &nan, term).unwrap(), 1, "NaN sorts last"); + assert_eq!(sql_cmp(&mut db, term, &nan).unwrap(), -1, "NaN sorts last"); + } + assert_eq!(sql_cmp(&mut db, &nan, &nan).unwrap(), 0, "NaN ties NaN"); + + let reals = [ + f32::NEG_INFINITY, + -2.5, + -0.0, + 0.0, + 0.5, + f32::MAX, + f32::INFINITY, + ]; + let real_terms = runtime.block_on(terms(&keyset, "RealOrdOre", &reals)); + assert_agree::(&mut db, "RealOrdOre", &reals, &real_terms); +} + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_compares_chained_text_terms_as_ore_rs_and_the_plaintexts_do() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + // Lower-case ASCII folds to itself, so the fold order is byte order. + let mut values = texts(); + values.sort(); + values.dedup(); + let refs: Vec<&str> = values.iter().map(String::as_str).collect(); + let text_terms = runtime.block_on(text_terms(&keyset, "TextOrdOre", &refs)); + let counts: std::collections::BTreeSet = text_terms + .iter() + .map(|t| u16::from_be_bytes([t[2], t[3]])) + .collect(); + assert!( + counts.len() >= 3, + "the values span padding buckets: {counts:?}" + ); + assert_agree::(&mut db, "TextOrdOre", &values, &text_terms); + + // Text that differs only in case and accents folds to one value. + let a = runtime.block_on(ob(&keyset, "TextOrdOre", frame("Café"))); + let b = runtime.block_on(ob(&keyset, "TextOrdOre", frame("cafe"))); + assert_eq!(sql_cmp(&mut db, &a, &b).unwrap(), 0); +} + +/// ore.rs's vectors for the σ-MMO hash bit +/// (`docs/reviews/2026-06-16-mmo-plpgsql-validation.sql` in ore.rs). +const HASH_VECTORS: [(&str, &str, i32); 12] = [ + ( + "00070e151c232a31383f464d545b6269", + "010e1b2835424f5c697683909daab7c4", + 1, + ), + ( + "1f262d343b424950575e656c737a8188", + "121f2c394653606d7a8794a1aebbc8d5", + 0, + ), + ( + "3e454c535a61686f767d848b9299a0a7", + "23303d4a5764717e8b98a5b2bfccd9e6", + 0, + ), + ( + "5d646b727980878e959ca3aab1b8bfc6", + "34414e5b6875828f9ca9b6c3d0ddeaf7", + 0, + ), + ( + "7c838a91989fa6adb4bbc2c9d0d7dee5", + "45525f6c798693a0adbac7d4e1eefb08", + 0, + ), + ( + "9ba2a9b0b7bec5ccd3dae1e8eff6fd04", + "5663707d8a97a4b1becbd8e5f2ff0c19", + 1, + ), + ( + "bac1c8cfd6dde4ebf2f900070e151c23", + "6774818e9ba8b5c2cfdce9f603101d2a", + 0, + ), + ( + "d9e0e7eef5fc030a11181f262d343b42", + "7885929facb9c6d3e0edfa0714212e3b", + 0, + ), + ( + "f8ff060d141b222930373e454c535a61", + "8996a3b0bdcad7e4f1fe0b1825323f4c", + 0, + ), + ( + "171e252c333a41484f565d646b727980", + "9aa7b4c1cedbe8f5020f1c293643505d", + 0, + ), + ( + "363d444b525960676e757c838a91989f", + "abb8c5d2dfecf90613202d3a4754616e", + 1, + ), + ( + "555c636a71787f868d949ba2a9b0b7be", + "bcc9d6e3f0fd0a1724313e4b5865727f", + 0, + ), +]; + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_hash_bit_matches_ore_rs_vectors() { + let mut db = connect(); + for (f, nonce, expected) in HASH_VECTORS { + let bit: i32 = db + .query_one( + "SELECT eql_v4_internal.ore_block_v2_hash_bit($1::bytea, $2::bytea)", + &[&unhex(f), &unhex(nonce)], + ) + .unwrap() + .get(0); + assert_eq!(bit, expected, "{f} under {nonce}"); + } +} + +fn raises(db: &mut postgres::Client, a: &[u8], b: &[u8], what: &str) -> String { + match sql_cmp(db, a, b) { + Ok(n) => panic!("{what}: compared to {n}, not refused"), + Err(e) => e + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_else(|| e.to_string()), + } +} + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_refuses_malformed_and_mismatched_terms() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let int = runtime.block_on(ob(&keyset, "IntegerOrdOre", frame(7i32))); + let big = runtime.block_on(ob(&keyset, "BigintOrdOre", frame(7i64))); + let text = runtime.block_on(ob(&keyset, "TextOrdOre", frame("alice"))); + + let mut cases: Vec<(&str, Vec)> = vec![ + ("truncated", int[..int.len() - 1].to_vec()), + ("one byte long", int.iter().copied().chain([0]).collect()), + ("empty", Vec::new()), + ("header alone", int[..4].to_vec()), + ]; + let mut version = int.clone(); + version[0] = 1; + cases.push(("wire version 1", version)); + let mut scheme = int.clone(); + scheme[1] = 4; + cases.push(("an unknown scheme", scheme)); + let mut symbol = int.clone(); + symbol[4] = 64; + cases.push(("a symbol of 64", symbol)); + let mut count = int.clone(); + count[3] += 1; + cases.push(("a block count its length does not have", count)); + let mut policy = text.clone(); + policy[4] = 3; + cases.push(("an unknown padding kind", policy)); + let mut bucket = text.clone(); + bucket[5..7].copy_from_slice(&64u16.to_be_bytes()); + cases.push(("a block count its padding cannot produce", bucket)); + for (what, bad) in &cases { + let message = raises(&mut db, bad, &int, what); + assert!( + message.contains("Malformed ORE block term"), + "{what}: {message}" + ); + let message = raises(&mut db, &text, bad, what); + assert!( + message.contains("Malformed ORE block term"), + "{what}: {message}" + ); + assert!( + !message.contains(&hex(bad)) || bad.is_empty(), + "{what}: the message names the length, never the bytes" + ); + } + + let message = raises(&mut db, &int, &text, "fixed against chained"); + assert!(message.contains("different schemes"), "{message}"); + let message = raises(&mut db, &int, &big, "6 blocks against 11"); + assert!(message.contains("different lengths"), "{message}"); + // The same text term under the no-padding policy, which admits any + // block count: well formed, but padded differently. + let mut unpadded = text.clone(); + unpadded[4..7].copy_from_slice(&[0, 0, 0]); + let message = raises(&mut db, &text, &unpadded, "two padding policies"); + assert!(message.contains("different policies"), "{message}"); + + // A value holds exactly one term. + let error = db + .query_one( + "SELECT eql_v4_internal.compare_ore_block_v2_terms(\ + ARRAY[ROW($1::bytea), ROW($1::bytea)]::eql_v4_internal.ore_block_v2_term[], \ + ARRAY[ROW($1::bytea)]::eql_v4_internal.ore_block_v2_term[])", + &[&int], + ) + .unwrap_err(); + assert!( + error + .as_db_error() + .unwrap() + .message() + .contains("exactly one term"), + "{error}" + ); +} + +/// The longest chained term EQL v4 admits: 1366 blocks, a 1,024-byte string +/// (the longest Stack Encrypt writes, marker byte included). +const MAX_CHAINED_BLOCKS: usize = 1366; + +/// A term of `count` all-zero blocks with the header given, well formed in +/// every respect but the header. +fn zero_term(header: &[u8], count: usize) -> Vec { + let mut term = header.to_vec(); + term.resize(header.len() + 25 * count + 16, 0); + term +} + +fn chained_header(kind: u8, policy: u16, count: usize) -> Vec { + let count = u16::try_from(count).unwrap().to_be_bytes(); + let policy = policy.to_be_bytes(); + vec![2, 3, count[0], count[1], kind, policy[0], policy[1]] +} + +/// Needs an EQL v4 installation. For every padding kind, a spread of +/// policies around each power-of-two bucket and every block count up to +/// past the bound, EQL v4 admits a term exactly when ore-rs does and the +/// term has between one block and [`MAX_CHAINED_BLOCKS`]; for the +/// fixed-width scheme, exactly when ore-rs does. ore-rs parses a chained +/// term of no blocks, which Stack Encrypt never writes (text and bytes go +/// in behind a marker byte), and EQL v4 refuses it. The terms are built in the database, from +/// the header fields alone, so the check does not ship each one. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_admits_the_block_counts_ore_rs_does_up_to_the_bound() { + let mut db = connect(); + db.batch_execute( + "CREATE FUNCTION pg_temp.admits(t bytea) RETURNS boolean LANGUAGE plpgsql AS $$ + BEGIN + PERFORM eql_v4_internal.ore_block_v2_scheme(t); + RETURN true; + EXCEPTION WHEN raise_exception THEN + RETURN false; + END $$", + ) + .unwrap(); + + let mut policies: Vec = vec![0, 1, 2, 3, 5, 7, 49151, 49152, 65535]; + for k in 2..=15 { + let bucket = 1u32 << k; + for p in [bucket - 1, bucket, bucket + 1, bucket * 3 / 4] { + policies.push(u16::try_from(p).unwrap()); + } + } + policies.sort_unstable(); + policies.dedup(); + let mut counts: Vec = (0..=MAX_CHAINED_BLOCKS + 40).collect(); + counts.extend([2048, 2731, 2732, 21846, 43691, 65535]); + + let mut cases: Vec<(u8, u16, usize)> = Vec::new(); + for kind in 0..=3u8 { + let kind_policies: &[u16] = match kind { + 1 | 2 => &policies, + _ => &[0, 1, 16], + }; + for &policy in kind_policies { + for &count in &counts { + cases.push((kind, policy, count)); + } + } + } + + let kinds: Vec = cases.iter().map(|c| i32::from(c.0)).collect(); + let pols: Vec = cases.iter().map(|c| i32::from(c.1)).collect(); + let cnts: Vec = cases.iter().map(|c| c.2 as i32).collect(); + let admitted: Vec = db + .query( + "SELECT pg_temp.admits( + '\\x0203'::bytea || substr(int4send(c), 3, 2) || substr(int4send(k), 4, 1) + || substr(int4send(p), 3, 2) || decode(repeat('00', 25 * c + 16), 'hex')) + FROM unnest($1::int[], $2::int[], $3::int[]) WITH ORDINALITY AS x(k, p, c, n) + ORDER BY n", + &[&kinds, &pols, &cnts], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(admitted.len(), cases.len()); + + let mut both = 0; + for (&(kind, policy, count), &sql) in cases.iter().zip(&admitted) { + let term = zero_term(&chained_header(kind, policy, count), count); + let ore_rs = OreBlockTerm::::from_bytes(&term).is_ok(); + let expected = ore_rs && (1..=MAX_CHAINED_BLOCKS).contains(&count); + assert_eq!( + sql, expected, + "chained kind {kind}, policy {policy}, {count} blocks: ore-rs says {ore_rs}" + ); + both += usize::from(sql); + } + // The grid reaches both answers, and past the bound ore-rs alone admits. + assert!(both > 0 && both < cases.len()); + assert!(cases + .iter() + .zip(&admitted) + .any(|(&(_, _, count), &sql)| count > MAX_CHAINED_BLOCKS + && !sql + && OreBlockTerm::::from_bytes(&zero_term( + &chained_header(0, 0, count), + count + )) + .is_ok())); + + for count in 0..=20usize { + let header = [2, 2, 0, u8::try_from(count).unwrap()]; + let term = zero_term(&header, count); + let sql: bool = db + .query_one("SELECT pg_temp.admits($1)", &[&term]) + .unwrap() + .get(0); + let ore_rs = OreBlockTerm::::from_bytes(&term).is_ok(); + assert_eq!(sql, ore_rs, "fixed-width, {count} blocks"); + } +} + +/// Needs an EQL v4 installation. The longest text Stack Encrypt orders, +/// 1023 bytes, is a term of exactly [`MAX_CHAINED_BLOCKS`] blocks and +/// compares; the same shape one block longer, which ore-rs alone would +/// accept, is refused before any block is compared. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_compares_the_longest_term_and_refuses_one_past_it() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let longest = "z".repeat(1023); + let shorter = format!("{}y", "z".repeat(1022)); + let a = runtime.block_on(ob(&keyset, "TextOrdOre", frame(longest.as_str()))); + let b = runtime.block_on(ob(&keyset, "TextOrdOre", frame(shorter.as_str()))); + assert_eq!( + usize::from(u16::from_be_bytes([a[2], a[3]])), + MAX_CHAINED_BLOCKS + ); + assert_eq!( + as_ordering(sql_cmp(&mut db, &a, &b).unwrap()), + Ordering::Greater + ); + assert_eq!( + as_ordering(sql_cmp(&mut db, &b, &a).unwrap()), + Ordering::Less + ); + assert_eq!( + as_ordering(sql_cmp(&mut db, &a, &a).unwrap()), + Ordering::Equal + ); + + let count = MAX_CHAINED_BLOCKS + 1; + let over = zero_term(&chained_header(0, 0, count), count); + assert!( + OreBlockTerm::::from_bytes(&over).is_ok(), + "ore-rs admits it" + ); + let unpadded = zero_term( + &chained_header(0, 0, MAX_CHAINED_BLOCKS), + MAX_CHAINED_BLOCKS, + ); + for (x, y) in [(&over, &unpadded), (&unpadded, &over)] { + let message = raises(&mut db, x, y, "one block past the bound"); + assert!(message.contains("Malformed ORE block term"), "{message}"); + } +} + +/// Needs an EQL v4 installation. Rows written through the by-name dispatch +/// and queried through a btree index on the order term: `<`, `>=`, `=` and +/// `ORDER BY` find exactly the rows the plaintexts say. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_answers_the_orderings_through_a_btree_index() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + db.batch_execute( + "CREATE TEMP TABLE v4_ore_block (id int PRIMARY KEY, \ + level public.eql_v4_integer_ord_ore, title public.eql_v4_text_ord_ore);", + ) + .unwrap(); + let rows: [(i32, i32, &str); 6] = [ + (1, -5, "alice"), + (2, 0, "bob"), + (3, 0, "bob"), + (4, 7, "carol"), + (5, 64, "carola"), + (6, i32::MAX, "zoe"), + ]; + for (id, level, title) in rows { + let (level, title) = runtime.block_on(async { + ( + stored(&keyset, "IntegerOrdOre", frame(level)) + .await + .to_string(), + stored(&keyset, "TextOrdOre", frame(title)) + .await + .to_string(), + ) + }); + db.execute( + "INSERT INTO v4_ore_block VALUES ($1, $2::text::jsonb::public.eql_v4_integer_ord_ore, \ + $3::text::jsonb::public.eql_v4_text_ord_ore)", + &[&id, &level, &title], + ) + .unwrap(); + } + db.batch_execute( + "CREATE INDEX ON v4_ore_block USING btree (eql_v4.ord_term_ore(level)); \ + CREATE INDEX ON v4_ore_block USING btree (eql_v4.ord_term_ore(title)); \ + ANALYZE v4_ore_block; SET enable_seqscan = off;", + ) + .unwrap(); + let probe = |name: &str, value: ValueFrame| -> String { + runtime.block_on(async { + let bytes = targets::query(name, &common::index_of(name), &keyset, &label(), value) + .unwrap() + .await + .unwrap(); + String::from_utf8(bytes).unwrap() + }) + }; + let level = probe("IntegerOrdOre", frame(0i32)); + let title = probe("TextOrdOre", frame("bob")); + for (column, domain, probe) in [ + ("level", "query_integer_ord_ore", &level), + ("title", "query_text_ord_ore", &title), + ] { + let ids = |db: &mut postgres::Client, op: &str| -> Vec { + let sql = format!( + "SELECT id FROM v4_ore_block WHERE {column} {op} $1::text::jsonb::eql_v4.{domain} ORDER BY id" + ); + db.query(&sql, &[probe]) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect() + }; + assert_eq!(ids(&mut db, "<"), [1], "{column} <"); + assert_eq!(ids(&mut db, ">="), [2, 3, 4, 5, 6], "{column} >="); + assert_eq!(ids(&mut db, "="), [2, 3], "{column} ="); + let plan: Vec = db + .query( + &format!( + "EXPLAIN SELECT id FROM v4_ore_block WHERE {column} < $1::text::jsonb::eql_v4.{domain}" + ), + &[probe], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert!( + plan.iter().any(|line| line.contains("Index")), + "{column}: the btree index answers `<`: {plan:#?}" + ); + let ordered: Vec = db + .query( + &format!("SELECT id FROM v4_ore_block ORDER BY eql_v4.ord_term_ore({column}), id"), + &[], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!(ordered, [1, 2, 3, 4, 5, 6], "{column}: ORDER BY"); + } +} diff --git a/packages/eql/tests/encryption/tests/v4_range.rs b/packages/eql/tests/encryption/tests/v4_range.rs index e5038b01d..29eb6e375 100644 --- a/packages/eql/tests/encryption/tests/v4_range.rs +++ b/packages/eql/tests/encryption/tests/v4_range.rs @@ -6,70 +6,75 @@ //! value the single-bound query domain for the same term holds under its term //! key (`ob` for block ORE, `op` for CLLW OPE). //! -//! EQL v4's ordered types are not producible yet: EQL compares block-ORE and -//! CLLW-OPE terms, and Stack Encrypt does not derive those for EQL (#1132, -//! #1236). So the order terms here come from cipherstash-client's ORE and OPE -//! indexers, the code that writes the terms EQL's comparators are built for, -//! run offline over `FakeDataKeySource`'s index key as -//! `ste_vec_selectors.rs` does. The rest of each stored value (`v`, `i`, `c`, -//! and `hm` for text) is a real Stack Encrypt `Integer` or `TextEq`. Nothing -//! here derives an order term itself. +//! The order terms, stored and in each bound, are the engine's: Stack Encrypt +//! derives the block ORE (`ob`) and CLLW OPE (`op`) terms of EQL v4's ordered +//! types, so a row is a real `IntegerOrdOre`, `TextOrdOpe`, …, and a bound is +//! the term the single-bound query of that type carries. Nothing here derives +//! an order term itself. The ordered date, timestamp and numeric types are +//! not producible (the engine derives no order term for those kinds), so the +//! smoke test ranges over the families whose ordered types are. //! //! The PostgreSQL tests need an EQL v4 installation (`mise run //! test:encryption:postgres`); the others need no database. mod common; -use cipherstash_client::encryption::{IndexTerm, OpeIndexer, OreIndexer, Plaintext}; -use cipherstash_client::zerokms; +use eql_bindings::encryption::targets; use eql_bindings::v4::{self, integer, text, DomainType}; -use eql_bindings::Identifier; use serde_json::{json, Value}; -use stack_encrypt::NonEmpty; -use stack_kms::{FakeDataKeySource, IndexKeySource}; - -fn column() -> NonEmpty { - Identifier::for_column("users", "age").unwrap() -} +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::target::IndexSpec; +use stack_encrypt::{Encrypt, KeysetCipher, Label}; fn identifier() -> Value { json!({"t": "users", "c": "age"}) } -/// `FakeDataKeySource`'s index key for the default keyset, as the key type -/// cipherstash-client's indexers take. -async fn index_key() -> zerokms::IndexKey { - let (_, key) = FakeDataKeySource::new().load_index_key(None).await.unwrap(); - zerokms::IndexKey::from(*key.key()) +fn label() -> Label { + Label::new(["users", "age"]).unwrap() } -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() } -/// The `ob` value cipherstash-client writes for `plaintext`: one hex block -/// for a number, one per block for text. `query` asks for the query form. -fn ob(key: &zerokms::IndexKey, plaintext: Plaintext, query: bool) -> Value { - let term = if query { - OreIndexer.encrypt_for_query(&plaintext, key) - } else { - OreIndexer.encrypt(&plaintext, key) - }; - match term.unwrap() { - IndexTerm::OreFull(bytes) => json!([hex(&bytes)]), - IndexTerm::OreArray(blocks) => { - Value::Array(blocks.iter().map(|b| Value::String(hex(b))).collect()) - } - other => panic!("not an ORE term: {other:?}"), - } +/// The value the engine seals for `value` as the EQL type `name`, as the +/// JSON text a driver binds: its order term (`ob` or `op`) is what `between` +/// compares a bound against. +async fn stored( + keyset: &KeysetCipher<'_, common::Observed>, + name: &str, + value: ValueFrame, +) -> String { + let bytes = targets::encrypt(name, keyset, &label(), value) + .unwrap() + .await + .unwrap(); + String::from_utf8(bytes).unwrap() } -/// The `op` value cipherstash-client writes for `plaintext`. -fn op(key: &zerokms::IndexKey, plaintext: Plaintext) -> Value { - match OpeIndexer.encrypt(&plaintext, key).unwrap() { - IndexTerm::OpeFixed(bytes) | IndexTerm::OpeVariable(bytes) => Value::String(hex(&bytes)), - other => panic!("not an OPE term: {other:?}"), - } +/// The order term the engine derives for `value` queried as the EQL type +/// `name` through `index` (`"ore_block"` for a block ORE bound, `"ope"` for a +/// CLLW OPE one): what a range holds under `lo` and `hi`, the same value the +/// single-bound query domain holds under `ob` or `op`. +async fn bound( + keyset: &KeysetCipher<'_, common::Observed>, + name: &str, + index: &str, + value: ValueFrame, +) -> Value { + let spec = IndexSpec::parse(index).unwrap(); + let bytes = targets::query(name, &spec, keyset, &label(), value) + .unwrap() + .await + .unwrap(); + let mut query: Value = serde_json::from_slice(&bytes).unwrap(); + let term = match index { + "ore_block" => "ob", + "ope" => "op", + other => panic!("no range over the {other} index"), + }; + query[term].take() } /// A range over the bounds `lo` and `hi`, read through the binding `T` (so @@ -85,15 +90,18 @@ fn range( #[tokio::test] async fn range_bindings_hold_two_bounds_of_their_term_and_refuse_anything_else() { - let key = index_key().await; - let (lo, hi) = (Plaintext::Int(Some(10)), Plaintext::Int(Some(30))); + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let lo_ob = bound(&keyset, "IntegerOrdOre", "ore_block", frame(10i32)).await; + let hi_ob = bound(&keyset, "IntegerOrdOre", "ore_block", frame(30i32)).await; + let lo_op = bound(&keyset, "IntegerOrd", "ope", frame(10i32)).await; + let hi_op = bound(&keyset, "IntegerOrd", "ope", frame(30i32)).await; // Each bound is what the single-bound query operand holds as `ob`: the // same value reads as either. (A block-ORE term carries a nonce, so two // derivations of one value differ; one derivation is used for both.) - let lo_ob = ob(&key, lo.clone(), true); let ore: integer::IntegerRangeOreQuery = serde_json::from_value(json!({ - "v": 4, "i": identifier(), "lo": lo_ob, "hi": ob(&key, hi.clone(), true), + "v": 4, "i": identifier(), "lo": lo_ob, "hi": hi_ob, })) .unwrap(); let single: integer::IntegerOrdOreQuery = @@ -120,16 +128,16 @@ async fn range_bindings_hold_two_bounds_of_their_term_and_refuse_anything_else() let ope: integer::IntegerRangeOpeQuery = serde_json::from_value(json!({ "v": 4, "i": identifier(), - "lo": op(&key, lo.clone()), "hi": op(&key, hi.clone()), + "lo": lo_op.clone(), "hi": hi_op.clone(), })) .unwrap(); let single: integer::IntegerOrdOpeQuery = serde_json::from_value(json!({ - "v": 4, "i": identifier(), "op": op(&key, lo.clone()), + "v": 4, "i": identifier(), "op": lo_op.clone(), })) .unwrap(); assert_eq!(ope.lo, single.op); - let good = json!({"v": 4, "i": identifier(), "lo": op(&key, lo.clone()), "hi": op(&key, hi)}); + let good = json!({"v": 4, "i": identifier(), "lo": lo_op, "hi": hi_op}); let refused = |change: &dyn Fn(&mut Value)| { let mut value = good.clone(); change(&mut value); @@ -252,13 +260,9 @@ fn sqlstate(error: &postgres::Error) -> String { const INTEGERS: [i32; 12] = [-1000, -5, -1, 0, 1, 7, 10, 20, 29, 30, 31, 1000]; /// Integer rows in all three ordered column domains, each value's real -/// Stack Encrypt `Integer` (`v`, `i`, `c`) with cipherstash-client's order -/// term added: `ob` for `_ord_ore`, `op` for `_ord` and `_ord_ope`. -fn integer_table( - runtime: &tokio::runtime::Runtime, - db: &mut postgres::Client, - key: &zerokms::IndexKey, -) { +/// The engine's `IntegerOrdOre`, `IntegerOrd` and `IntegerOrdOpe` values of +/// each of `INTEGERS`: `ob` for `_ord_ore`, `op` for `_ord` and `_ord_ope`. +fn integer_table(runtime: &tokio::runtime::Runtime, db: &mut postgres::Client) { let (cipher, _) = runtime.block_on(common::cipher()); let keyset = cipher.default_keyset(); db.batch_execute( @@ -271,15 +275,7 @@ fn integer_table( ) .unwrap(); for n in INTEGERS { - let stored: integer::Integer = runtime - .block_on(async { keyset.encrypt_as(&n, column()).await }) - .unwrap(); - let base = serde_json::to_value(&stored).unwrap(); - let with = |k: &str, term: Value| { - let mut v = base.clone(); - v[k] = term; - v.to_string() - }; + let row = |name: &str| runtime.block_on(stored(&keyset, name, frame(n))); db.execute( "INSERT INTO v4_int_range VALUES ($1, $2::text::jsonb::public.eql_v4_integer_ord_ore, @@ -287,9 +283,9 @@ fn integer_table( $4::text::jsonb::public.eql_v4_integer_ord_ope)", &[ &n, - &with("ob", ob(key, Plaintext::Int(Some(n)), false)), - &with("op", op(key, Plaintext::Int(Some(n)))), - &with("op", op(key, Plaintext::Int(Some(n)))), + &row("IntegerOrdOre"), + &row("IntegerOrd"), + &row("IntegerOrdOpe"), ], ) .unwrap(); @@ -302,21 +298,22 @@ fn integer_table( #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_finds_integers_in_an_inclusive_range() { let runtime = runtime(); - let key = runtime.block_on(index_key()); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); { let mut db = connect(); - integer_table(&runtime, &mut db, &key); + integer_table(&runtime, &mut db); let ore = |lo: i32, hi: i32| { range::( - ob(&key, Plaintext::Int(Some(lo)), true), - ob(&key, Plaintext::Int(Some(hi)), true), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(lo))), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(hi))), ) }; let ope = |lo: i32, hi: i32| { range::( - op(&key, Plaintext::Int(Some(lo))), - op(&key, Plaintext::Int(Some(hi))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(lo))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(hi))), ) }; let mut found = |column: &str, domain: &str, range: &str| -> Vec { @@ -405,7 +402,6 @@ fn postgres_between_finds_integers_in_an_inclusive_range() { #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_finds_text_in_an_inclusive_range() { let runtime = runtime(); - let key = runtime.block_on(index_key()); let (cipher, _) = runtime.block_on(common::cipher()); let keyset = cipher.default_keyset(); let words = [ @@ -419,15 +415,9 @@ fn postgres_between_finds_text_in_an_inclusive_range() { ]; let mut rows = Vec::new(); for (id, word) in words.iter().enumerate() { - let stored: text::TextEq = runtime - .block_on(async { keyset.encrypt_as(&word.to_string(), column()).await }) - .unwrap(); - let base = serde_json::to_value(&stored).unwrap(); - let mut ore = base.clone(); - ore["ob"] = ob(&key, Plaintext::Text(Some(word.to_string())), false); - let mut ope = base; - ope["op"] = op(&key, Plaintext::Text(Some(word.to_string()))); - rows.push((id as i32, ore.to_string(), ope.to_string())); + let ore = runtime.block_on(stored(&keyset, "TextOrdOre", frame(word.to_string()))); + let ope = runtime.block_on(stored(&keyset, "TextOrdOpe", frame(word.to_string()))); + rows.push((id as i32, ore, ope)); } { let mut db = connect(); @@ -448,7 +438,6 @@ fn postgres_between_finds_text_in_an_inclusive_range() { ) .unwrap(); } - let text = |s: &str| Plaintext::Text(Some(s.to_owned())); let mut found = |column: &str, domain: &str, range: &str| -> Vec { db.query( &format!( @@ -472,15 +461,28 @@ fn postgres_between_finds_text_in_an_inclusive_range() { ]; for (lo, hi, want) in cases { let ore = range::( - ob(&key, text(lo), true), - ob(&key, text(hi), true), + runtime.block_on(bound( + &keyset, + "TextOrdOre", + "ore_block", + frame(lo.to_string()), + )), + runtime.block_on(bound( + &keyset, + "TextOrdOre", + "ore_block", + frame(hi.to_string()), + )), ); assert_eq!( found("ore", "query_text_range_ore", &ore), want, "ORE [{lo}, {hi}]" ); - let ope = range::(op(&key, text(lo)), op(&key, text(hi))); + let ope = range::( + runtime.block_on(bound(&keyset, "TextOrd", "ope", frame(lo.to_string()))), + runtime.block_on(bound(&keyset, "TextOrd", "ope", frame(hi.to_string()))), + ); assert_eq!( found("ope", "query_text_range_ope", &ope), want, @@ -507,10 +509,11 @@ fn postgres_between_finds_text_in_an_inclusive_range() { #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_uses_the_order_term_index() { let runtime = runtime(); - let key = runtime.block_on(index_key()); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); { let mut db = connect(); - integer_table(&runtime, &mut db, &key); + integer_table(&runtime, &mut db); db.batch_execute( "CREATE INDEX v4_int_range_ore ON v4_int_range USING btree (eql_v4.ord_term_ore(ore)); CREATE INDEX v4_int_range_ope ON v4_int_range USING btree (eql_v4.ord_term(ope)); @@ -519,12 +522,12 @@ fn postgres_between_uses_the_order_term_index() { ) .unwrap(); let ore = range::( - ob(&key, Plaintext::Int(Some(10)), true), - ob(&key, Plaintext::Int(Some(30)), true), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(10i32))), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(30i32))), ); let ope = range::( - op(&key, Plaintext::Int(Some(10))), - op(&key, Plaintext::Int(Some(30))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(10i32))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(30i32))), ); for (column, domain, index, range) in [ ("ore", "query_integer_range_ore", "v4_int_range_ore", &ore), @@ -597,10 +600,13 @@ fn postgres_between_uses_the_order_term_index() { #[test] #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_range_domains_check_their_shape() { - let key = runtime().block_on(index_key()); - let (lo, hi) = (Plaintext::Int(Some(1)), Plaintext::Int(Some(2))); - let ore = json!({"v": 4, "i": identifier(), "lo": ob(&key, lo.clone(), true), "hi": ob(&key, hi.clone(), true)}); - let ope = json!({"v": 4, "i": identifier(), "lo": op(&key, lo), "hi": op(&key, hi)}); + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let term = + |name: &str, index: &str, n: i32| runtime.block_on(bound(&keyset, name, index, frame(n))); + let ore = json!({"v": 4, "i": identifier(), "lo": term("IntegerOrdOre", "ore_block", 1), "hi": term("IntegerOrdOre", "ore_block", 2)}); + let ope = json!({"v": 4, "i": identifier(), "lo": term("IntegerOrd", "ope", 1), "hi": term("IntegerOrd", "ope", 2)}); { let mut db = connect(); let cast = |db: &mut postgres::Client, value: &Value, domain: &str| { @@ -667,50 +673,34 @@ fn postgres_range_domains_check_their_shape() { } } -/// Five plaintexts of `family`, strictly ascending, as cipherstash-client's -/// indexers take them: what the smoke test stores and ranges over. A new -/// ordered family must add a row here, or the test fails naming it. -fn ascending(family: &str) -> [Plaintext; 5] { - use chrono::{DateTime, NaiveDate}; - use rust_decimal::Decimal; - let date = |y, m, d| Plaintext::NaiveDate(NaiveDate::from_ymd_opt(y, m, d)); - let at = |secs| Plaintext::Timestamp(DateTime::from_timestamp(secs, 0)); - let dec = |n, scale| Plaintext::Decimal(Some(Decimal::new(n, scale))); - let text = |s: &str| Plaintext::Text(Some(s.to_owned())); - match family { - "integer" => [-20, -1, 0, 7, 100].map(|n| Plaintext::Int(Some(n))), - "smallint" => [-300, -1, 0, 5, 300].map(|n| Plaintext::SmallInt(Some(n))), - "bigint" => [i64::MIN + 1, -1, 0, 1 << 40, i64::MAX].map(|n| Plaintext::BigInt(Some(n))), - "date" => [ - date(1969, 12, 31), - date(1970, 1, 1), - date(2000, 2, 29), - date(2024, 6, 1), - date(2099, 12, 31), - ], - "timestamp" => [-86_400, 0, 951_782_400, 1_717_200_000, 4_102_444_800].map(at), - "numeric" => [ - dec(-15, 1), - dec(0, 0), - dec(25, 2), - dec(314_159, 5), - dec(1000, 0), - ], - "text" => ["apple", "banana", "cherry", "damson", "elderberry"].map(text), - // `real` values are exact in f32; both float widths encrypt as f64. - "real" => [-2.5, -0.5, 0.0, 1.25, 1e10].map(|f| Plaintext::Float(Some(f))), - "double" => [-1e300, -1.0, 0.0, 2.5, 1e300].map(|f| Plaintext::Float(Some(f))), +/// Five values of `family`, strictly ascending: what the smoke test stores +/// and ranges over. `None` for a family whose ordered types the engine does +/// not produce. A new ordered family must add a row here, or the test fails +/// naming it. +fn ascending(family: &str) -> Option<[Value; 5]> { + Some(match family { + "integer" => [-20, -1, 0, 7, 100].map(Value::from), + "smallint" => [-300, -1, 0, 5, 300].map(Value::from), + "bigint" => [i64::MIN + 1, -1, 0, 1 << 40, i64::MAX].map(Value::from), + "text" => ["apple", "banana", "cherry", "damson", "elderberry"].map(Value::from), + // `real` values are exact in f32; both float widths order as f64. + "real" => [-2.5, -0.5, 0.0, 1.25, 1e10].map(Value::from), + "double" => [-1e300, -1.0, 0.0, 2.5, 1e300].map(Value::from), + "date" | "timestamp" | "numeric" => return None, other => panic!("no smoke-test values for the ordered family {other:?}: add them here"), - } + }) } -/// The text a smoke-test value seals as for its `c` (and, for text, its -/// `hm` and `bf`): `between` never reads them, and most ordered families -/// have no producible type to seal the value itself. -fn display(plaintext: &Plaintext) -> String { - match plaintext { - Plaintext::Text(Some(s)) => s.clone(), - other => format!("{other:?}"), +/// `value` as the plaintext `family`'s types seal. +fn frame_of(family: &str, value: &Value) -> ValueFrame { + match family { + "integer" => frame(i32::try_from(value.as_i64().unwrap()).unwrap()), + "smallint" => frame(i16::try_from(value.as_i64().unwrap()).unwrap()), + "bigint" => frame(value.as_i64().unwrap()), + "real" => frame(value.as_f64().unwrap() as f32), + "double" => frame(value.as_f64().unwrap()), + "text" => frame(value.as_str().unwrap().to_owned()), + other => panic!("no plaintext for the ordered family {other:?}"), } } @@ -725,24 +715,48 @@ fn display(plaintext: &Plaintext) -> String { #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_smoke_test_over_every_ordered_family() { let runtime = runtime(); - let key = runtime.block_on(index_key()); let (cipher, _) = runtime.block_on(common::cipher()); let keyset = cipher.default_keyset(); let mut db = connect(); let ranges = v4::all_range_query(); let mut checked = 0; + let mut skipped = Vec::new(); for family in eql_domains::CATALOG .iter() .filter(|f| !f.range_terms().is_empty()) { - let values = ascending(family.name); let table = format!("v4_smoke_{}", family.name); let columns: Vec<&eql_domains::Domain> = family .domains .iter() .filter(|d| d.terms.iter().any(|t| t.range_suffix().is_some())) .collect(); + // A family's ordered types are producible together or not at all: + // the engine derives no order term for a date, a timestamp or a + // decimal, so there is no row of those to range over. + let producible: Vec = columns + .iter() + .map(|d| { + targets::target(&d.rust_struct_name(family.name)).is_some_and(|t| t.producible) + }) + .collect(); + if producible.iter().all(|p| !p) { + assert!(ascending(family.name).is_none(), "{}", family.name); + skipped.push(family.name); + continue; + } + assert!(producible.iter().all(|p| *p), "{}", family.name); + let values = ascending(family.name) + .unwrap_or_else(|| panic!("{}: its ordered types are producible", family.name)); + let type_of = |suffix: &str| { + family + .domains + .iter() + .find(|d| d.name == suffix) + .unwrap_or_else(|| panic!("{} has no {suffix} domain", family.name)) + .rust_struct_name(family.name) + }; db.batch_execute(&format!( "CREATE TEMP TABLE {table} (id int PRIMARY KEY, {})", columns @@ -754,37 +768,14 @@ fn postgres_between_smoke_test_over_every_ordered_family() { .unwrap(); for (id, value) in values.iter().enumerate() { - let text = display(value); - let (eq, matched) = runtime.block_on(async { - let eq: text::TextEq = keyset.encrypt_as(&text, column()).await.unwrap(); - let matched = match family.name { - "text" => Some( - keyset - .encrypt_as::<_, text::TextMatch>(&text, column()) - .await - .unwrap(), - ), - _ => None, - }; - (eq, matched) - }); - let eq = serde_json::to_value(&eq).unwrap(); let payloads: Vec = columns .iter() .map(|d| { - let mut p = json!({"v": eq["v"], "i": eq["i"], "c": eq["c"]}); - for term in d.terms { - match term { - eql_domains::Term::Hm => p["hm"] = eq["hm"].clone(), - eql_domains::Term::Bloom => { - let m = matched.as_ref().expect("only text has a match term"); - p["bf"] = serde_json::to_value(m).unwrap()["bf"].clone(); - } - eql_domains::Term::Ore => p["ob"] = ob(&key, value.clone(), false), - eql_domains::Term::Ope => p["op"] = op(&key, value.clone()), - } - } - p.to_string() + runtime.block_on(stored( + &keyset, + &d.rust_struct_name(family.name), + frame_of(family.name, value), + )) }) .collect(); let placeholders: Vec = columns @@ -816,13 +807,16 @@ fn postgres_between_smoke_test_over_every_ordered_family() { } for term in family.range_terms() { - let bound = |v: &Plaintext| match term { - eql_domains::Term::Ore => ob(&key, v.clone(), true), - eql_domains::Term::Ope => op(&key, v.clone()), + let (name, index) = match term { + eql_domains::Term::Ore => (type_of("ord_ore"), "ore_block"), + eql_domains::Term::Ope => (type_of("ord"), "ope"), other => unreachable!("{other:?} has no range"), }; + let bound_of = |v: &Value| { + runtime.block_on(bound(&keyset, &name, index, frame_of(family.name, v))) + }; let domain = family.range_query_name(term); - let range = json!({"v": 4, "i": identifier(), "lo": bound(&values[1]), "hi": bound(&values[3])}); + let range = json!({"v": 4, "i": identifier(), "lo": bound_of(&values[1]), "hi": bound_of(&values[3])}); // The binding for the domain accepts the range. ranges .iter() @@ -857,11 +851,13 @@ fn postgres_between_smoke_test_over_every_ordered_family() { if family.name == "text" { let range = |term: eql_domains::Term| { - let bound = |v: &Plaintext| match term { - eql_domains::Term::Ore => ob(&key, v.clone(), true), - _ => op(&key, v.clone()), + let (name, index) = match term { + eql_domains::Term::Ore => ("TextOrdOre", "ore_block"), + _ => ("TextOrd", "ope"), }; - json!({"v": 4, "i": identifier(), "lo": bound(&values[1]), "hi": bound(&values[3])}) + let bound_of = + |v: &Value| runtime.block_on(bound(&keyset, name, index, frame_of("text", v))); + json!({"v": 4, "i": identifier(), "lo": bound_of(&values[1]), "hi": bound_of(&values[3])}) .to_string() }; db.batch_execute(&format!( @@ -908,8 +904,13 @@ fn postgres_between_smoke_test_over_every_ordered_family() { db.batch_execute("RESET enable_seqscan;").unwrap(); } } - // 8 numeric and temporal families with 3 overloads each, and text's 5. - assert_eq!(checked, 29, "every between overload was exercised"); + // The 5 numeric families with 3 overloads each, and text's 5. + assert_eq!( + checked, 20, + "every between overload of a producible type was exercised" + ); + skipped.sort_unstable(); + assert_eq!(skipped, ["date", "numeric", "timestamp"]); } /// Needs an EQL v4 installation. The range parameter must be cast to its @@ -995,7 +996,7 @@ impl Drop for NonSuperuserDatabase { /// Needs an EQL v4 installation, and a superuser on `EQL_TEST_DATABASE_URL` /// who can create a role and a database. A non-superuser install skips the -/// `ore_block_256` btree opclass, and `ore_fallback.sql` disables the +/// block ORE btree opclass, and `ore_fallback.sql` disables the /// single-bound ORE domains. Every ORE range domain is disabled the same way, /// naming its family's OPE range, which stays usable. The bundle is the one /// `mise run test:encryption:postgres` builds and installs. @@ -1017,7 +1018,7 @@ fn postgres_a_non_superuser_install_disables_the_ore_ranges() { "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_opclass c JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = 'eql_v4_internal.ore_block_256'::pg_catalog.regtype)", + AND c.opcintype = 'eql_v4_internal.ore_block_v2'::pg_catalog.regtype)", &[], ) .unwrap() diff --git a/packages/eql/tests/encryption/tests/v4_text_search.rs b/packages/eql/tests/encryption/tests/v4_text_search.rs new file mode 100644 index 000000000..bc90bb75e --- /dev/null +++ b/packages/eql/tests/encryption/tests/v4_text_search.rs @@ -0,0 +1,582 @@ +//! EQL v4's `TextSearch`: one column answering `=`, `<` and `@@`, whose +//! query carries only the term its operator reads (ADR-0002, amended +//! 2026-10-19). +//! +//! A stored value carries all three terms, each the one the single-index +//! type stores (`TextEq`'s `hm`, `TextOrd`'s `op`, `TextMatch`'s `bf`). +//! Text too short to cut a 3-gram stores the empty filter, where `TextMatch` +//! refuses it, so `=` and `<` find it and no `@@` query does. A query is built +//! through the projection of the index that answers its operator, so `=` and +//! `<` on such text work and a match query on it is refused. Against an EQL +//! v4 database: a query with any one term is a `query_text_search`, one with +//! none is not, and each operator refuses an operand without its term (or +//! with it `null`, or an empty filter) rather than matching no row. + +mod common; + +use eql_bindings::encryption::targets; +use eql_bindings::v4::text::{ + TextSearch, TextSearchEqualityQuery, TextSearchMatchQuery, TextSearchOrderQuery, + TextSearchQuery, +}; +use eql_bindings::Identifier; +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::sem::TermError; +use stack_encrypt::target::IndexSpec; +use stack_encrypt::{Encrypt, Label, NonEmpty}; + +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() +} + +fn column() -> NonEmpty { + Identifier::for_column("people", "name").unwrap() +} + +fn label() -> Label { + Label::new(["people", "name"]).unwrap() +} + +/// The index a query on `TextSearch` is asked through, by its key. +fn index(key: &str) -> IndexSpec { + IndexSpec::parse(key).unwrap() +} + +/// The stored JSON the named type writes for `text`. +async fn stored( + keyset: &stack_encrypt::KeysetCipher<'_, common::Observed>, + name: &str, + text: &str, +) -> serde_json::Value { + let bytes = targets::encrypt(name, keyset, &label(), frame(text)) + .unwrap() + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +/// The `TextSearch` query for `text` through the index `key`. +async fn query( + keyset: &stack_encrypt::KeysetCipher<'_, common::Observed>, + key: &str, + text: &str, +) -> Result { + let bytes = targets::query("TextSearch", &index(key), keyset, &label(), frame(text)) + .unwrap() + .await?; + Ok(serde_json::from_slice(&bytes).unwrap()) +} + +fn keys(value: &serde_json::Value) -> Vec<&str> { + let mut keys: Vec<&str> = value + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + keys +} + +#[tokio::test] +async fn a_stored_value_carries_the_three_single_index_terms() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let search = stored(&keyset, "TextSearch", "Alice").await; + assert_eq!(keys(&search), ["bf", "c", "hm", "i", "op", "v"]); + assert_eq!(search["v"], 4); + assert_eq!(search["hm"], stored(&keyset, "TextEq", "Alice").await["hm"]); + assert_eq!( + search["op"], + stored(&keyset, "TextOrd", "Alice").await["op"] + ); + assert_eq!( + search["bf"], + stored(&keyset, "TextMatch", "Alice").await["bf"] + ); + let opened = targets::decrypt( + "TextSearch", + &cipher, + &label(), + search.to_string().as_bytes(), + ) + .unwrap() + .await + .unwrap(); + assert_eq!(opened.risky_ref(), frame("Alice").risky_ref()); +} + +/// Text too short to cut a 3-gram has no match tokens. `TextSearch` stores +/// it with the empty filter and the same `hm` and `op` as `TextEq` and +/// `TextOrd`, so `=` and `<` find it and no match query does. `TextMatch`, +/// which has no other term to find it by, still refuses it. +#[tokio::test] +async fn short_text_stores_the_empty_filter_where_text_match_refuses_it() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + for text in ["Al", "", " "] { + let search = stored(&keyset, "TextSearch", text).await; + assert_eq!(keys(&search), ["bf", "c", "hm", "i", "op", "v"], "{text:?}"); + assert_eq!(search["bf"], serde_json::json!([]), "{text:?}"); + assert_eq!( + search["hm"], + stored(&keyset, "TextEq", text).await["hm"], + "{text:?}" + ); + assert_eq!( + search["op"], + stored(&keyset, "TextOrd", text).await["op"], + "{text:?}" + ); + let opened = targets::decrypt( + "TextSearch", + &cipher, + &label(), + search.to_string().as_bytes(), + ) + .unwrap() + .await + .unwrap(); + assert_eq!(opened.risky_ref(), frame(text).risky_ref(), "{text:?}"); + + let refused = targets::encrypt("TextMatch", &keyset, &label(), frame(text)) + .unwrap() + .await + .unwrap_err(); + assert!( + matches!( + refused, + stack_encrypt::Error::Term(TermError::EmptyTermText) + ), + "TextMatch {text:?}: {refused:?}" + ); + } +} + +/// Each index's query carries its term alone, and it is the stored one. +#[tokio::test] +async fn a_query_carries_only_the_term_its_operator_reads() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let search = stored(&keyset, "TextSearch", "Alice").await; + for (key, term) in [("eq", "hm"), ("ope", "op"), ("match", "bf")] { + let q = query(&keyset, key, "Alice").await.unwrap(); + assert_eq!(keys(&q), { + let mut expected = vec!["i", "v", term]; + expected.sort_unstable(); + expected + }); + assert_eq!(q[term], search[term], "{key}: the stored {term}"); + assert_eq!(q["v"], 4); + } + // An index the type does not carry is refused before anything runs. + let Err(refused) = targets::query("TextSearch", &IndexSpec::Ore, &keyset, &label(), frame("x")) + else { + panic!("an ore query on TextSearch was dispatched"); + }; + assert!( + matches!( + refused, + targets::TargetError::NoIndex { + name: "TextSearch", + index: "ore" + } + ), + "{refused:?}" + ); +} + +/// `=` and `<` on text shorter than one 3-gram: the query carries `hm` or +/// `op`, never an empty filter. A match query on it is refused, as a +/// `TextMatchQuery` is, rather than sent as a filter that matches every row. +#[tokio::test] +async fn short_text_is_queried_by_equality_and_order_and_refused_for_match() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let eq = query(&keyset, "eq", "Al").await.unwrap(); + assert_eq!(keys(&eq), ["hm", "i", "v"]); + assert!(eq["hm"].as_str().is_some_and(|hm| !hm.is_empty())); + let ord = query(&keyset, "ope", "Al").await.unwrap(); + assert_eq!(keys(&ord), ["i", "op", "v"]); + assert!(ord["op"].as_str().is_some_and(|op| !op.is_empty())); + // The order term of "Al" sorts before "Alice"'s: a prefix first. + let alice = stored(&keyset, "TextSearch", "Alice").await; + assert!(ord["op"].as_str().unwrap() < alice["op"].as_str().unwrap()); + + let refused = query(&keyset, "match", "Al").await.unwrap_err(); + assert!( + matches!( + refused, + stack_encrypt::Error::Term(TermError::EmptyTermText) + ), + "{refused:?}" + ); +} + +/// The typed path: each projection derives its one term and converts into +/// the `TextSearchQuery` the by-name dispatch writes, byte for byte. +#[tokio::test] +async fn the_typed_projections_are_what_the_dispatch_writes() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let text = "Al".to_owned(); + let eq: TextSearchEqualityQuery = keyset.encrypt_as(&text, column()).await.unwrap(); + let ord: TextSearchOrderQuery = keyset.encrypt_as(&text, column()).await.unwrap(); + for (typed, key) in [ + (TextSearchQuery::from(eq), "eq"), + (TextSearchQuery::from(ord), "ope"), + ] { + assert_eq!( + serde_json::to_value(&typed).unwrap(), + query(&keyset, key, "Al").await.unwrap(), + "{key}" + ); + } + let matched: TextSearchMatchQuery = keyset + .encrypt_as(&"Alice".to_owned(), column()) + .await + .unwrap(); + let matched = TextSearchQuery::from(matched); + assert!(matched.hm.is_none() && matched.op.is_none() && matched.bf.is_some()); + assert_eq!( + serde_json::to_value(&matched).unwrap(), + query(&keyset, "match", "Alice").await.unwrap() + ); + // A query with no term deserializes, as the JSON Schema allows; it is + // the database that refuses it. + let bare: TextSearchQuery = serde_json::from_value(serde_json::json!({ + "v": 4, "i": {"t": "people", "c": "name"} + })) + .unwrap(); + assert!(bare.hm.is_none() && bare.op.is_none() && bare.bf.is_none()); + + // The stored value round-trips through the typed path too. + let stored: TextSearch = keyset + .encrypt_as(&"Alice".to_owned(), column()) + .await + .unwrap(); + let opened: String = cipher.decrypt_as(stored, column().into()).await.unwrap(); + assert_eq!(opened, "Alice"); + // And short text through the typed path stores the empty filter, as the + // dispatch does. + let short: TextSearch = keyset.encrypt_as(&text, column()).await.unwrap(); + assert!(short.bf.0 .0.is_empty()); + assert_eq!( + serde_json::to_value(&short.bf).unwrap(), + serde_json::json!([]) + ); + let opened: String = cipher.decrypt_as(short, column().into()).await.unwrap(); + assert_eq!(opened, "Al"); +} + +// ---- PostgreSQL ------------------------------------------------------------- + +fn connect() -> postgres::Client { + postgres::Client::connect( + &std::env::var("EQL_TEST_DATABASE_URL").expect("EQL_TEST_DATABASE_URL"), + postgres::NoTls, + ) + .unwrap() +} + +/// SQLSTATE `check_violation`. +const CHECK_VIOLATION: &str = "23514"; +/// SQLSTATE `invalid_parameter_value`. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// The rows whose `name` the SQL's predicate selects, the operand bound as +/// `$1`. +fn ids(db: &mut postgres::Client, sql: &str, operand: &serde_json::Value) -> Vec { + let mut ids: Vec = db + .query(sql, &[&operand.to_string()]) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect(); + ids.sort_unstable(); + ids +} + +/// The error the SQL raises with the operand bound as `$1`. +fn refusal(db: &mut postgres::Client, sql: &str, operand: &serde_json::Value) -> postgres::Error { + match db.query(sql, &[&operand.to_string()]) { + Ok(rows) => panic!("{sql}: answered {} rows, expected a refusal", rows.len()), + Err(error) => error, + } +} + +fn sqlstate(error: &postgres::Error) -> &str { + error.code().map_or("", |code| code.code()) +} + +/// Needs an EQL v4 installation. Rows written through the by-name dispatch, +/// queried with operands that carry one term each: `=`, `<`, `>` and `@@` +/// find the rows the plaintexts say, two-letter and empty text, stored with +/// the empty filter, included on `=` and `<` and found by no `@@`; a +/// `query_text_search` holds any one term and refuses none; and each +/// operator refuses an operand without its term, naming the term, never the +/// operand. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_answers_each_operator_through_its_own_term() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + db.batch_execute( + "CREATE TEMP TABLE v4_search (id int PRIMARY KEY, name public.eql_v4_text_search);", + ) + .unwrap(); + // 4 and 5 are too short for a 3-gram: stored with `"bf": []`, which the + // domain's CHECK accepts. + let names = [(1, "alice"), (2, "Bob"), (3, "carol"), (4, "Al"), (5, "")]; + for (id, name) in names { + let value = runtime.block_on(stored(&keyset, "TextSearch", name)); + db.execute( + "INSERT INTO v4_search VALUES ($1, $2::text::jsonb::public.eql_v4_text_search)", + &[&id, &value.to_string()], + ) + .unwrap(); + } + let q = |key: &str, text: &str| runtime.block_on(query(&keyset, key, text)).unwrap(); + let select = |predicate: &str| format!("SELECT id FROM v4_search WHERE {predicate}"); + let operand = "$1::text::jsonb::eql_v4.query_text_search"; + + // Each operator through its own term. + assert_eq!( + ids( + &mut db, + &select(&format!("name = {operand}")), + &q("eq", "Bob") + ), + [2] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name <> {operand}")), + &q("eq", "Bob") + ), + [1, 3, 4, 5] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name < {operand}")), + &q("ope", "bob") + ), + [1, 4, 5] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name >= {operand}")), + &q("ope", "bob") + ), + [2, 3] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name @@ {operand}")), + &q("match", "aro") + ), + [3] + ); + // Two-letter text, which has no match term, on `=` and `<`/`>`: stored + // and queried. + assert_eq!( + ids( + &mut db, + &select(&format!("name = {operand}")), + &q("eq", "Al") + ), + [4] + ); + assert_eq!( + ids(&mut db, &select(&format!("name = {operand}")), &q("eq", "")), + [5] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name > {operand}")), + &q("ope", "Bo") + ), + [2, 3] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name < {operand}")), + &q("ope", "Bo") + ), + [1, 4, 5] + ); + // The function forms, as PostgREST calls them. + assert_eq!( + ids( + &mut db, + &select(&format!("eql_v4.lt(name, {operand})")), + &q("ope", "bob") + ), + [1, 4, 5] + ); + // No match query finds a stored empty filter: `matches` needs the stored + // filter to contain the query's, and a query's is never empty. + for needle in ["ali", "alic", "Bob", "aro", "xyz"] { + let found = ids( + &mut db, + &select(&format!("name @@ {operand}")), + &q("match", needle), + ); + assert!( + !found.contains(&4) && !found.contains(&5), + "{needle:?} matched an empty filter: {found:?}" + ); + } + assert_eq!( + ids( + &mut db, + &select(&format!("name @@ {operand}")), + &q("match", "ali") + ), + [1] + ); + + // The domain holds a query with any one term, and refuses one with none. + let full = q("eq", "x"); + for key in ["eq", "ope", "match"] { + db.query(&format!("SELECT {operand}"), &[&q(key, "xyz").to_string()]) + .unwrap_or_else(|e| panic!("{key}: {e}")); + } + let mut bare = full.clone(); + bare.as_object_mut().unwrap().remove("hm"); + let error = refusal(&mut db, &format!("SELECT {operand}"), &bare); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + // The v3 operand is untouched: it still requires every term. + let mut v3 = q("eq", "x"); + v3["v"] = 3.into(); + // Only when the v3 bundle is installed beside v4; this database may + // hold v4 alone. + let has_v3: bool = db + .query_one( + "SELECT to_regtype('eql_v3.query_text_search') IS NOT NULL", + &[], + ) + .unwrap() + .get(0); + if has_v3 { + let error = refusal( + &mut db, + "SELECT $1::text::jsonb::eql_v3.query_text_search", + &v3, + ); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + } + + // Each operator refuses an operand without the term it reads, naming + // the term and never the operand. + for (predicate, key, missing) in [ + ("name = {}", "ope", "hm"), + ("name <> {}", "match", "hm"), + ("name < {}", "eq", "op"), + ("name >= {}", "match", "op"), + ("name @@ {}", "eq", "bf"), + ("name @@ {}", "ope", "bf"), + ("eql_v4.eq_term({}) IS NOT NULL", "ope", "hm"), + ("eql_v4.ord_term({}) IS NOT NULL", "eq", "op"), + ("eql_v4.match_term({}) IS NOT NULL", "ope", "bf"), + ] { + let sql = select(&predicate.replace("{}", operand)); + let without = q(key, "carol"); + let error = refusal(&mut db, &sql, &without); + assert_eq!(sqlstate(&error), INVALID_PARAMETER_VALUE, "{sql}: {error}"); + let message = error + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_default(); + assert!( + message.contains(&format!( + "eql_v4.query_text_search value has no \"{missing}\" term" + )), + "{sql}: {message}" + ); + let carried = without.as_object().unwrap(); + for term in ["hm", "op", "bf"].iter().filter_map(|t| carried.get(*t)) { + let shown = term.to_string(); + let shown = shown.trim_matches('"'); + assert!( + !message.contains(shown), + "{sql}: the message quotes the operand" + ); + } + } + + // A term that is JSON `null`, or a filter with no positions, is as + // missing as an absent one. Neither comes from Stack Encrypt, which + // leaves an absent term out and never builds an empty query filter; a + // hand-built operand is refused rather than compared as NULL (no row), + // or as the empty filter only the short-text rows store. + let mut null_hm = q("eq", "carol"); + null_hm["hm"] = serde_json::Value::Null; + let mut empty_bf = q("match", "carol"); + empty_bf["bf"] = serde_json::json!([]); + for (predicate, operand_value, missing) in [ + ("name = {}", &null_hm, "hm"), + ("eql_v4.eq_term({}) IS NOT NULL", &null_hm, "hm"), + ("name @@ {}", &empty_bf, "bf"), + ("eql_v4.match_term({}) IS NOT NULL", &empty_bf, "bf"), + ] { + let sql = select(&predicate.replace("{}", operand)); + let error = refusal(&mut db, &sql, operand_value); + assert_eq!(sqlstate(&error), INVALID_PARAMETER_VALUE, "{sql}: {error}"); + let message = error + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_default(); + assert!( + message.contains(&format!( + "eql_v4.query_text_search value has no \"{missing}\" term" + )), + "{sql}: {message}" + ); + } + + // The ORE twin follows the same rule, and a present `ob` is still a + // non-empty array. Its order term is EQL's block ORE, which the engine + // does not derive, so only the equality and match operands are real. + // An installer that could not create the ORE operator class (no + // superuser) poisons the ORE domains, and then there is nothing to test. + let poisoned: bool = db + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'eql_ore_unavailable' \ + AND contypid = 'eql_v4.query_text_search_ore'::regtype)", + &[], + ) + .unwrap() + .get(0); + if poisoned { + return; + } + let ore = "$1::text::jsonb::eql_v4.query_text_search_ore"; + db.query(&format!("SELECT {ore}"), &[&q("eq", "xyz").to_string()]) + .unwrap(); + db.query(&format!("SELECT {ore}"), &[&q("match", "xyz").to_string()]) + .unwrap(); + let mut empty_ob = q("eq", "xyz"); + empty_ob["ob"] = serde_json::json!([]); + let error = refusal(&mut db, &format!("SELECT {ore}"), &empty_ob); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + let error = refusal(&mut db, &format!("SELECT {ore}"), &bare); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + let error = refusal( + &mut db, + &format!("SELECT eql_v4.ord_term_ore({ore}) IS NOT NULL"), + &q("eq", "xyz"), + ); + assert_eq!(sqlstate(&error), INVALID_PARAMETER_VALUE, "{error}"); +} diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 84f0c32a6..0eb777c1c 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -210,7 +210,7 @@ async fn eq_terms( value: ValueFrame, ) -> (serde_json::Value, serde_json::Value) { let stored: serde_json::Value = by_name(keyset, name, value.clone()).await; - let query = targets::query(name, keyset, &label(), value) + let query = targets::query(name, &common::index_of(name), keyset, &label(), value) .unwrap() .await .unwrap(); @@ -431,34 +431,78 @@ async fn every_producible_type_dispatches_by_name() { match target.query { None => { - let error = refused(targets::query(name, &keyset, &label(), sample(kind))); + let error = refused(targets::query( + name, + &common::index_of(name), + &keyset, + &label(), + sample(kind), + )); assert!( matches!(error, TargetError::NoQuery { .. }), "{name}: {error}" ); } Some(_) => { - let query = targets::query(name, &keyset, &label(), sample(kind)) - .unwrap() - .await - .unwrap(); - let query: serde_json::Value = serde_json::from_slice(&query).unwrap(); - assert!( - query.get("c").is_none(), - "{name}: a query carries no ciphertext" - ); - if target.indexes.contains(&"eq") { - assert_eq!( - query["hm"], json["hm"], - "{name}: the query probes the stored term" - ); - } - if target.indexes.contains(&"ope") { - assert!(json["op"].is_string(), "{name} stores an op"); - assert_eq!( - query["op"], json["op"], - "{name}: the query compares the stored order term" + // Asked through each index the type carries, the query + // carries that index's term: the stored one, save a block + // ORE term, which is encrypted under a fresh nonce each + // time and so only has the stored one's shape. The search + // types' queries carry it alone (EQL v4's search query + // terms are optional); every other twin carries all of its + // terms whatever the index. + let term_of = |key: &str| match key { + "eq" => "hm", + "ope" => "op", + "match" => "bf", + "ore_block" => "ob", + other => panic!("{name}: no term for index {other}"), + }; + for key in target.indexes { + let index = stack_encrypt::target::IndexSpec::parse(key).unwrap(); + let query = targets::query(name, &index, &keyset, &label(), sample(kind)) + .unwrap() + .await + .unwrap(); + let query: serde_json::Value = serde_json::from_slice(&query).unwrap(); + assert!( + query.get("c").is_none(), + "{name}: a query carries no ciphertext" ); + let term = term_of(key); + assert!(json[term].is_string() || json[term].is_array(), "{name}"); + if term == "ob" { + let one_hex = |value: &serde_json::Value| { + value + .as_array() + .is_some_and(|a| a.len() == 1 && a[0].is_string()) + }; + assert!( + one_hex(&json[term]) && one_hex(&query[term]), + "{name} via {key}: one block ORE term each" + ); + assert_eq!( + query[term][0].as_str().unwrap().len(), + json[term][0].as_str().unwrap().len(), + "{name} via {key}: the query's term is the stored one's length" + ); + } else { + assert_eq!( + query[term], json[term], + "{name} via {key}: the query carries the stored term" + ); + } + let carried: Vec<&str> = target + .indexes + .iter() + .map(|key| term_of(key)) + .filter(|term| query.get(term).is_some()) + .collect(); + if matches!(name, "TextSearch" | "TextSearchOre") { + assert_eq!(carried, [term], "{name} via {key}: its term alone"); + } else { + assert_eq!(carried.len(), target.indexes.len(), "{name} via {key}"); + } } } } @@ -481,7 +525,13 @@ async fn storage_only_text_round_trips_through_both_entry_points() { let opened: String = cipher.decrypt_as(named, column().into()).await.unwrap(); assert_eq!(opened, "note"); // A storage-only type answers no query. - let error = refused(targets::query("Text", &keyset, &label(), frame("note"))); + let error = refused(targets::query( + "Text", + &common::index_of("Text"), + &keyset, + &label(), + frame("note"), + )); assert!( matches!(error, TargetError::NoQuery { name: "Text" }), "{error}" @@ -1258,7 +1308,7 @@ async fn the_v4_query_fixture_is_what_the_dispatch_derives() { let keyset = &keyset; async move { let label = Label::new(["profiles", column]).unwrap(); - let bytes = targets::query(name, keyset, &label, value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label, value) .unwrap() .await .unwrap(); diff --git a/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs b/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs index 3e93e86d0..7abf4480b 100644 --- a/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs +++ b/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs @@ -1,12 +1,14 @@ //! EQL v3 and v4 installed side by side in one database (ADR-0002). //! //! The two bundles are the same SQL under two names: `eql_v3` for terms written -//! by cipherstash-client, `eql_v4` for terms written by Stack Encrypt. These -//! tests pin what the separate names do and do not enforce when both are +//! by cipherstash-client, `eql_v4` for terms written by Stack Encrypt, plus the +//! files confined to v4 with `-- ONLY EQL VERSION: 4` (the JSON tree family). +//! These tests pin what the separate names do and do not enforce when both are //! installed: //! -//! - both install without clashing, each owns its own domains, and each -//! version's `lints()` stays clean with the other installed; +//! - both install without clashing, each owns its own domains (v4 every one +//! of v3's and the v4-only ones besides), and each version's `lints()` +//! stays clean with the other installed; //! - a payload of the wrong version fails the domain CHECK on insert; //! - the function form across versions (`eql_v3.eq(v3col, v4term)`) is //! ambiguous, so it errors, and so is `OPERATOR(public.=)` between a v3 and @@ -25,6 +27,8 @@ //! version set to 4: the two bundles share their SQL, so only the version //! decides which domain admits it. +use std::collections::BTreeSet; + use anyhow::Result; use sqlx::PgPool; @@ -122,6 +126,22 @@ fn sqlstate(err: &sqlx::Error) -> Option { } } +/// The public domains each version owns, by name: `eql_v3_*` and `eql_v4_*`. +async fn public_domains(pool: &PgPool) -> Result<(BTreeSet, BTreeSet)> { + let names: Vec = sqlx::query_scalar( + "SELECT typname::text FROM pg_type + WHERE typnamespace = 'public'::regnamespace AND typtype = 'd' + AND (typname LIKE 'eql\\_v3\\_%' OR typname LIKE 'eql\\_v4\\_%') + ORDER BY 1", + ) + .fetch_all(pool) + .await?; + let (v3, v4): (BTreeSet, BTreeSet) = names + .into_iter() + .partition(|name| name.starts_with("eql_v3_")); + Ok((v3, v4)) +} + #[sqlx::test(fixtures(path = "../fixtures", scripts("eql_v3_integer")))] async fn v3_and_v4_install_side_by_side(pool: PgPool) -> Result<()> { install_v4(&pool).await?; @@ -136,15 +156,33 @@ async fn v3_and_v4_install_side_by_side(pool: PgPool) -> Result<()> { ["eql_v3", "eql_v3_internal", "eql_v4", "eql_v4_internal"] ); - let (v3, v4): (i64, i64) = sqlx::query_as( - "SELECT count(*) FILTER (WHERE typname LIKE 'eql\\_v3\\_%'), - count(*) FILTER (WHERE typname LIKE 'eql\\_v4\\_%') - FROM pg_type WHERE typnamespace = 'public'::regnamespace AND typtype = 'd'", - ) - .fetch_one(&pool) - .await?; - assert!(v3 > 0, "v3 installed no public domains"); - assert_eq!(v3, v4, "each version owns the same set of public domains"); + // Every public domain v3 owns, v4 owns under its own prefix. v4 also + // owns the domains of the files confined to it with `-- ONLY EQL + // VERSION: 4` (`tasks/build.sh`): today the JSON tree family, which Stack + // Encrypt writes and cipherstash-client does not. Those are the only + // domains v4 may have that v3 lacks, and v3 may own nothing v4 lacks. + let (v3, v4) = public_domains(&pool).await?; + assert!(!v3.is_empty(), "v3 installed no public domains"); + let to_v3 = |name: &String| name.replacen("eql_v4_", "eql_v3_", 1); + let missing_from_v4: Vec<&str> = v3 + .iter() + .filter(|name| !v4.iter().any(|v4_name| to_v3(v4_name) == **name)) + .map(String::as_str) + .collect(); + assert!( + missing_from_v4.is_empty(), + "v3 owns public domains v4 does not: {missing_from_v4:?}" + ); + let v4_only: Vec<&str> = v4 + .iter() + .filter(|name| !v3.contains(&to_v3(name))) + .map(String::as_str) + .collect(); + assert_eq!( + v4_only, + ["eql_v4_json_tree", "eql_v4_json_tree_entry"], + "the only public domains v4 owns and v3 does not are the JSON tree family" + ); let (a, b): (String, String) = sqlx::query_as("SELECT eql_v3.version(), eql_v4.version()") .fetch_one(&pool) @@ -322,3 +360,102 @@ async fn eq_term_across_versions_compares_text(pool: PgPool) -> Result<()> { ); Ok(()) } + +/// SQLSTATE `invalid_parameter_value`. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// A text search query operand of `version` built from the first fixture row +/// whose filter is not empty, minus the ciphertext, the block ORE term and +/// the keys in `drop`. +fn search_operand(version: u8, drop: &[&str]) -> String { + let mut operand = format!( + "jsonb_set((SELECT payload - 'c' - 'ob' FROM fixtures.eql_v3_text \ + WHERE jsonb_array_length(payload -> 'bf') > 0 ORDER BY id LIMIT 1), '{{v}}', '{version}')" + ); + for key in drop { + operand = format!("({operand} - '{key}')"); + } + format!("({operand})::eql_v{version}.query_text_search") +} + +/// The two versions part on the text search query operand (ADR-0002, +/// amended 2026-10-19). v3's is cipherstash-client's and needs `hm`, `op` +/// and `bf`. v4's needs any one of them, since Stack Encrypt sends only the +/// term its operator reads, and each v4 operator refuses an operand without +/// its term instead of comparing a NULL and matching no row. +#[sqlx::test(fixtures(path = "../fixtures", scripts("eql_v3_text")))] +async fn a_v4_search_query_needs_one_term_and_a_v3_one_needs_all(pool: PgPool) -> Result<()> { + install_v4(&pool).await?; + + let one_term: [&[&str]; 3] = [&["op", "bf"], &["hm", "bf"], &["hm", "op"]]; + sqlx::query(&format!("SELECT {}", search_operand(3, &[]))) + .execute(&pool) + .await?; + for drop in one_term { + let err = sqlx::query(&format!("SELECT {}", search_operand(3, drop))) + .execute(&pool) + .await + .expect_err("a v3 search query needs every term"); + assert_eq!( + sqlstate(&err).as_deref(), + Some(CHECK_VIOLATION), + "v3 without {drop:?}: {err}" + ); + } + for drop in std::iter::once(&[][..]).chain(one_term) { + sqlx::query(&format!("SELECT {}", search_operand(4, drop))) + .execute(&pool) + .await + .unwrap_or_else(|e| panic!("v4 without {drop:?}: {e}")); + } + let err = sqlx::query(&format!( + "SELECT {}", + search_operand(4, &["hm", "op", "bf"]) + )) + .execute(&pool) + .await + .expect_err("a v4 search query needs a term"); + assert_eq!(sqlstate(&err).as_deref(), Some(CHECK_VIOLATION), "{err}"); + + sqlx::raw_sql( + "CREATE TABLE t4s (c public.eql_v4_text_search); + INSERT INTO t4s SELECT jsonb_set(payload - 'ob', '{v}', '4')::public.eql_v4_text_search + FROM fixtures.eql_v3_text WHERE jsonb_array_length(payload -> 'bf') > 0;", + ) + .execute(&pool) + .await?; + for (op, term, rest) in [ + ("=", "hm", ["op", "bf"]), + ("<=", "op", ["hm", "bf"]), + ("@@", "bf", ["hm", "op"]), + ] { + // With its term, the operator answers: the operand's own row. + let n: i64 = sqlx::query_scalar(&format!( + "SELECT count(*) FROM t4s WHERE c {op} {}", + search_operand(4, &rest) + )) + .fetch_one(&pool) + .await?; + assert!(n >= 1, "c {op} with only {term} finds its own row"); + // Without it, the operator refuses, naming the term. + let err = sqlx::query(&format!( + "SELECT count(*) FROM t4s WHERE c {op} {}", + search_operand(4, &[term]) + )) + .execute(&pool) + .await + .expect_err("an operator refuses an operand without its term"); + assert_eq!( + sqlstate(&err).as_deref(), + Some(INVALID_PARAMETER_VALUE), + "c {op} without {term}: {err}" + ); + assert!( + err.to_string().contains(&format!( + "eql_v4.query_text_search value has no \"{term}\" term" + )), + "c {op} without {term}: {err}" + ); + } + Ok(()) +} diff --git a/packages/eql/tests/timing/ore_block_v2/README.md b/packages/eql/tests/timing/ore_block_v2/README.md new file mode 100644 index 000000000..f90bbc6f1 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/README.md @@ -0,0 +1,108 @@ +# EQL v4 block ORE comparator timing harness + +Checks that `eql_v4_internal.compare_ore_block_v2_terms`, the comparator +behind every ordered block ORE operator and operator class in EQL v4, takes +the same time wherever its operands first differ, and whether they differ at +all. EQL v4 compares ore-rs's 6-bit block ORE (the fixed-width scheme and +the chained one), a different format from EQL v3's under the same `ob` key, +with a comparator of its own. + +## Why it matters + +The database operator gains nothing from the comparator's timing: they hold +the stored ciphertexts and can run the comparator on any two of them. The +party who gains is someone who can issue queries and time them without +reading the stored ciphertexts, such as an application user. Query results +already tell them the order of their value relative to stored ones. If the +comparison time depends on where the two values first differ, timing also +tells them how long a prefix they share. + +PL/pgSQL is not a constant-time environment. What it can give is a constant +*operation count*: for operands of one shape, the same statements and +function calls whatever their contents. The shape is the scheme and the two +block counts, which a term's length reveals anyway, so the comparator may +branch on it. The v4 comparator runs one statement per compared block, which +evaluates both of the block's comparisons and latches the first difference +arithmetically, runs the hash exactly once on every path (equal terms +included), and selects its result arithmetically. C-level effects inside +those statements (`memcmp` stopping early within 16 bytes, the offset a +`substr` reads at) remain, orders of magnitude smaller. + +## Running it + +From `packages/eql`: + +```sh +mise run build +# a scratch database with the EQL v4 bundle installed, e.g.: +psql "$DATABASE_URL" -f release/cipherstash-encrypt-v4.sql +DATABASE_URL=postgres://... tests/timing/ore_block_v2/run.sh # 100k samples per kind +DATABASE_URL=postgres://... tests/timing/ore_block_v2/run.sh 20000 # quick check +``` + +It needs `psql` and `cargo`, and no CipherStash credentials: `ctgen/` encrypts +with ore-rs under a random throwaway key. Run it on a quiet machine. It is not +a CI test, because timing on shared runners is too noisy to gate on. + +Everything it creates lives in an `ore_v2_timing` schema, dropped on exit. If +that schema already exists, it refuses to run rather than touch it. + +## Method + +`ctgen` produces real ore-rs 0.9.0 ciphertexts, one term per side, in four +kinds of two classes each, plus a correctness set: + +- **fixed**: `u64` terms of the fixed-width scheme (11 blocks), differing in + the first block against differing only in the last; +- **equal**: a `u64` against a second encryption of the same value (a + different nonce, no block differs), against a first-block difference; +- **chained**: 31-byte text of the chained scheme (a 32-byte bucket, 43 + blocks), differing in the first character against only the last; +- **prefix**: 10-byte text (a 16-byte bucket, 22 blocks) against 26-byte + text (a 32-byte bucket, 43 blocks) whose first 22 blocks equal it, so the + block counts settle the order, against the same shapes differing in the + first block; +- **checks**: 1,500 random pairs of either scheme with shared prefixes of + every length, whose expected sign is the plaintext order and, in `ctgen`, + ore-rs's own comparator's. The harness refuses to time a comparator that + gets any of them wrong, either way round. + +Sampling follows dudect, with three lessons from the ore.rs benches built in: + +1. **Both classes share one table, in random order.** With separate storage + per class, placement alone produced a timing difference. +2. **Several comparisons per sample** (16 for the fixed-width kinds, 8 for + the chained ones), so a sample spans many ticks of `clock_timestamp()`'s + microsecond clock. Each comparison's pair is drawn at random, with + replacement, from that class. +3. **The plain Welch t over every sample**, not a percentile-cropped maximum. + Cropping turns a difference in spread into a large t with no difference in + mean. + +Classes are chosen at random per sample, so drift affects both equally. `|t| < 5` over a hundred thousand samples +counts as no detectable difference. + +## Results + +PostgreSQL 17.5 run natively on macOS, Apple silicon, 100,000 samples per +kind, two runs. The machine was shared with other builds, which shows in +the difference between the runs' means; classes are drawn at random per +sample, so drift affects both equally. + +| kind | run 1 t | run 2 t | mean µs per sample (run 2, class 0 / 1) | +|---|---:|---:|---| +| fixed, first vs last block | −1.22 | +0.33 | 192.3 / 191.4 | +| fixed, equal vs first block | −0.65 | −0.40 | 168.1 / 168.1 | +| chained, first vs last block | +0.74 | +0.09 | 204.5 / 204.5 | +| chained, prefix vs first block | −2.33 | −4.01 | 159.2 / 159.5 | + +None passes |t| = 5. The prefix kind is negative on both runs: class 0, +where the block counts settle the order, is faster by about 0.3 µs per +sample of eight comparisons, some 40 ns a comparison, or 0.2% of one. That +is below the cost of a single statement, so it is a C-level effect of the +kind described above, not a skipped statement. A skipped hash or an early +exit would cost a statement or more per comparison. + +Cost: about 10.5 µs per fixed-width comparison of 11 blocks and about +20 to 25 µs per chained comparison of 22 to 43 blocks, whatever the +operands. diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/.gitignore b/packages/eql/tests/timing/ore_block_v2/ctgen/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/.gitignore @@ -0,0 +1 @@ +/target diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.lock b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.lock new file mode 100644 index 000000000..16d2dfbc8 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.lock @@ -0,0 +1,374 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", + "zeroize", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-block-v2-timing-ctgen" +version = "0.0.0" +dependencies = [ + "hex", + "ore-rs", + "rand", +] + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand", + "rand_chacha", + "subtle-ng", + "thiserror", + "zeroize", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.toml b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.toml new file mode 100644 index 000000000..c9a31f8ff --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.toml @@ -0,0 +1,21 @@ +# Ciphertext generator for the EQL v4 block ORE comparator timing harness +# (../run.sh). +# +# A DETACHED crate: the empty `[workspace]` table makes it its own workspace +# root, so it never joins the main workspace or its CI. It encrypts with ore-rs +# directly under a random throwaway key; the harness needs real ciphertexts but +# no CipherStash credentials, and nothing it produces is a test fixture. +[package] +name = "ore-block-v2-timing-ctgen" +version = "0.0.0" +edition = "2021" +publish = false + +[dependencies] +# The 6-bit fixed-width and chained schemes behind EQL v4's `ore_block_v2`; +# the same version as the root Cargo.lock. +ore-rs = { version = "=0.9.0", default-features = false } +rand = "0.8" +hex = "0.4" + +[workspace] diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/src/main.rs b/packages/eql/tests/timing/ore_block_v2/ctgen/src/main.rs new file mode 100644 index 000000000..203fbef0e --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/src/main.rs @@ -0,0 +1,146 @@ +//! ore-rs 6-bit block ORE ciphertexts for the EQL v4 comparator timing +//! harness (`../run.sh`). Tab-separated rows, one term per side, hex: +//! +//! fixed u64, 11 blocks; differ in block 0 / only block 10 +//! equal u64; a second encryption of a / differ in block 0 +//! chained 31-byte text, 43 blocks; differ in the first +//! character / only the last +//! prefix text in a 16-byte bucket against text in a +//! 32-byte bucket: every block a has equals b's +//! (the block counts settle it) / the first differs +//! check random pairs of either scheme with shared +//! prefixes; expected = sign(a cmp b) +//! +//! Every expected sign is the plaintext order, and ore-rs's own comparator +//! is asserted to agree before a row is written. +use std::cmp::Ordering; + +use ore_rs::scheme::bit2_w6::OreAes128Bit6ChaCha20; +use ore_rs::scheme::chained::OreAes128Bit6ChainedChaCha20; +use ore_rs::{OreCipher, OreEncrypt, OreOutput}; +use rand::Rng; + +fn sign(o: Ordering) -> i32 { + match o { + Ordering::Less => -1, + Ordering::Equal => 0, + Ordering::Greater => 1, + } +} + +fn lower(rng: &mut impl Rng, n: usize) -> String { + (0..n).map(|_| rng.gen_range(b'a'..=b'z') as char).collect() +} + +fn main() { + let mut rng = rand::thread_rng(); + let key: [u8; 16] = rng.gen(); + let fixed = OreAes128Bit6ChaCha20::init(&key).unwrap(); + let chained = OreAes128Bit6ChainedChaCha20::init(&key).unwrap(); + let ct = |x: u64| x.encrypt(&fixed).unwrap().to_bytes(); + let st = |s: &str| chained.encrypt_str(s).unwrap().to_bytes(); + let pairs: usize = std::env::args() + .nth(1) + .map(|s| s.parse().unwrap()) + .unwrap_or(256); + let row = |kind: &str, label: &str, a: &[u8], b: &[u8]| { + println!("{kind}\t{label}\t{}\t{}", hex::encode(a), hex::encode(b)); + }; + for _ in 0..pairs { + let x: u64 = rng.gen(); + row("fixed", "first", &ct(x), &ct(x ^ (1 << 63))); + row("fixed", "last", &ct(x), &ct(x ^ 1)); + row("equal", "same", &ct(x), &ct(x)); + row("equal", "first", &ct(x), &ct(x ^ (1 << 63))); + + let s = lower(&mut rng, 31); + let mut first = s.clone().into_bytes(); + first[0] = b'a' + (first[0] - b'a' + 1) % 26; + let mut last = s.clone().into_bytes(); + last[30] = b'a' + (last[30] - b'a' + 1) % 26; + row( + "chained", + "first", + &st(&s), + &st(std::str::from_utf8(&first).unwrap()), + ); + row( + "chained", + "last", + &st(&s), + &st(std::str::from_utf8(&last).unwrap()), + ); + + // 10 bytes pads with NULs to a 16-byte bucket: 22 blocks, the last + // holding two bits of byte 15 and four of byte 16. The longer text + // repeats the padding and puts a byte whose top four bits are clear + // at 16, so all 22 of the shorter text's blocks match it, and its + // 26 bytes take a 32-byte bucket. + let short = lower(&mut rng, 10); + let long = format!("{short}\0\0\0\0\0\0\u{5}{}", lower(&mut rng, 9)); + let (a, b) = (st(&short), st(&long)); + assert_eq!( + OreAes128Bit6ChainedChaCha20::compare_raw_slices(&a, &b), + Some(Ordering::Less) + ); + row("prefix", "prefix", &a, &b); + let mut other = long.clone().into_bytes(); + other[0] = b'a' + (other[0] - b'a' + 1) % 26; + row( + "prefix", + "first", + &a, + &st(std::str::from_utf8(&other).unwrap()), + ); + } + for i in 0..1500 { + if i % 2 == 0 { + let x: u64 = rng.gen(); + let keep = rng.gen_range(0..=64u32); + let mask = if keep == 0 { + 0 + } else { + u64::MAX << (64 - keep) + }; + let y = if i % 6 == 0 { + x + } else { + (x & mask) | (rng.gen::() & !mask) + }; + let (a, b) = (ct(x), ct(y)); + let expected = x.cmp(&y); + assert_eq!( + OreAes128Bit6ChaCha20::compare_raw_slices(&a, &b), + Some(expected) + ); + println!( + "check\t{}\t{}\t{}", + sign(expected), + hex::encode(&a), + hex::encode(&b) + ); + } else { + let len = rng.gen_range(0..=40); + let s = lower(&mut rng, len); + let shared = rng.gen_range(0..=s.len()); + let t = if i % 5 == 0 { + s.clone() + } else { + let tail = rng.gen_range(0..=40 - shared); + format!("{}{}", &s[..shared], lower(&mut rng, tail)) + }; + let (a, b) = (st(&s), st(&t)); + let expected = s.as_bytes().cmp(t.as_bytes()); + assert_eq!( + OreAes128Bit6ChainedChaCha20::compare_raw_slices(&a, &b), + Some(expected) + ); + println!( + "check\t{}\t{}\t{}", + sign(expected), + hex::encode(&a), + hex::encode(&b) + ); + } + } +} diff --git a/packages/eql/tests/timing/ore_block_v2/harness.sql b/packages/eql/tests/timing/ore_block_v2/harness.sql new file mode 100644 index 000000000..9778df9b4 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/harness.sql @@ -0,0 +1,98 @@ +-- EQL v4 block ORE comparator timing harness. Loaded by run.sh into a +-- database with the EQL v4 bundle installed, after run.sh has created the +-- `ore_v2_timing` schema and loaded the ctgen output into +-- ore_v2_timing.staging (`\copy` cannot take a psql variable). Everything +-- lives in `ore_v2_timing`, which run.sh creates and drops afterwards. + +SET search_path = ore_v2_timing, public; + +CREATE FUNCTION term_of(h text) + RETURNS eql_v4_internal.ore_block_v2_term[] + LANGUAGE sql IMMUTABLE AS $$ + SELECT ARRAY[ROW(decode(h, 'hex'))::eql_v4_internal.ore_block_v2_term] +$$; + +CREATE TABLE checks AS + SELECT label::int AS expected, term_of(a_hex) AS a, term_of(b_hex) AS b + FROM staging WHERE kind = 'check'; + +-- Timing pairs. Both classes of a kind share one table in random order, so +-- neither sits in a fixed region of memory: with separate per-class storage, +-- placement alone produced a timing difference in the ore.rs benches. +-- Class 1 is a difference in the first block. Class 0: +-- fixed, chained: a difference in the last block only; +-- equal: a second encryption of the same value (no block differs); +-- prefix: every block of the shorter term matches the longer's, so +-- the block counts settle the order. +CREATE TABLE pairs AS + SELECT kind, row_number() OVER (PARTITION BY kind ORDER BY random()) AS id, + (label = 'first')::int AS cls, term_of(a_hex) AS a, term_of(b_hex) AS b + FROM staging WHERE kind <> 'check'; + +-- dudect-style sampling. Each sample picks a class at random and times +-- `per_sample` comparisons of pairs of that class, each drawn at random with +-- replacement, through compare_ore_block_v2_terms over the composite (the +-- operator-class path). clock_timestamp() ticks in microseconds, so a sample +-- must span many of them. +CREATE FUNCTION bench(which_kind text, samples int, per_sample int) + RETURNS TABLE (cls int, us double precision) +AS $$ +DECLARE + arrs_a eql_v4_internal.ore_block_v2[]; + arrs_b eql_v4_internal.ore_block_v2[]; + idx0 int[]; + idx1 int[]; + picks int[]; + c int; + r int; + t0 timestamptz; + t1 timestamptz; +BEGIN + SELECT array_agg(ROW(p.a)::eql_v4_internal.ore_block_v2 ORDER BY p.id), + array_agg(ROW(p.b)::eql_v4_internal.ore_block_v2 ORDER BY p.id) + INTO arrs_a, arrs_b FROM ore_v2_timing.pairs p WHERE p.kind = which_kind; + SELECT array_agg(p.id::int ORDER BY p.id) FILTER (WHERE p.cls = 0), + array_agg(p.id::int ORDER BY p.id) FILTER (WHERE p.cls = 1) + INTO idx0, idx1 FROM ore_v2_timing.pairs p WHERE p.kind = which_kind; + FOR s IN 1..samples LOOP + c := (random() < 0.5)::int; + IF c = 0 THEN + picks := ARRAY(SELECT idx0[1 + floor(random() * cardinality(idx0))::int] FROM generate_series(1, per_sample)); + ELSE + picks := ARRAY(SELECT idx1[1 + floor(random() * cardinality(idx1))::int] FROM generate_series(1, per_sample)); + END IF; + t0 := clock_timestamp(); + FOREACH r IN ARRAY picks LOOP + PERFORM eql_v4_internal.compare_ore_block_v2_terms(arrs_a[r], arrs_b[r]); + END LOOP; + t1 := clock_timestamp(); + cls := c; + us := extract(epoch FROM t1 - t0) * 1e6; + RETURN NEXT; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +-- Uncropped Welch t (every sample counts), with per-class mean and median. +-- Report this, not a percentile-cropped maximum: cropping turns a difference +-- in spread into a large t with no difference in mean. +CREATE FUNCTION welch(label text, tbl regclass) + RETURNS TABLE (run text, n bigint, mean_0 numeric, mean_1 numeric, + median_0 numeric, median_1 numeric, t numeric) +AS $$ +BEGIN + RETURN QUERY EXECUTE format($q$ + WITH s AS ( + SELECT count(*) n, + count(*) FILTER (WHERE cls = 0) n0, count(*) FILTER (WHERE cls = 1) n1, + avg(us) FILTER (WHERE cls = 0) m0, avg(us) FILTER (WHERE cls = 1) m1, + stddev(us) FILTER (WHERE cls = 0) s0, stddev(us) FILTER (WHERE cls = 1) s1, + percentile_cont(0.5) WITHIN GROUP (ORDER BY us) FILTER (WHERE cls = 0) p0, + percentile_cont(0.5) WITHIN GROUP (ORDER BY us) FILTER (WHERE cls = 1) p1 + FROM %s) + SELECT %L::text, n, round(m0::numeric, 1), round(m1::numeric, 1), + round(p0::numeric, 1), round(p1::numeric, 1), + round(((m0 - m1) / sqrt(s0 * s0 / n0 + s1 * s1 / n1))::numeric, 2) + FROM s$q$, tbl, label); +END; +$$ LANGUAGE plpgsql; diff --git a/packages/eql/tests/timing/ore_block_v2/run.sh b/packages/eql/tests/timing/ore_block_v2/run.sh new file mode 100755 index 000000000..159d0ce24 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/run.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Timing harness for EQL v4's block ORE comparator (compare_ore_block_v2_terms). +# By hand, not CI: timing on shared runners is too noisy to gate on. +# +# DATABASE_URL=postgres://... tests/timing/ore_block_v2/run.sh [samples] +# +# DATABASE_URL must point at a database with the EQL v4 bundle installed (e.g. +# release/cipherstash-encrypt-v4.sql from `mise run build`). Needs psql and +# cargo. samples defaults to 100000 per kind, a few minutes in all. See +# README.md for the method and how to read the output. +set -euo pipefail + +samples="${1:-100000}" +here="$(cd "$(dirname "$0")" && pwd)" +: "${DATABASE_URL:?set DATABASE_URL to a database with the EQL v4 bundle installed}" + +# Everything lives in the ore_v2_timing schema, which is dropped on exit. +# Refuse to start if it already exists, so a run never drops a schema it did +# not make. A failed psql is reported as itself, not as an existing schema. +if ! exists="$(psql "$DATABASE_URL" -tA -c "SELECT to_regnamespace('ore_v2_timing') IS NOT NULL")"; then + echo "could not query the database at DATABASE_URL (see the psql error above)" >&2 + exit 1 +fi +if [ "$exists" != "f" ]; then + echo "schema ore_v2_timing already exists; not touching it" >&2 + exit 1 +fi + +tmp="$(mktemp -d)" +created=0 +cleanup() { + rm -rf "$tmp" + if [ "$created" = 1 ]; then + psql "$DATABASE_URL" -q -c "DROP SCHEMA ore_v2_timing CASCADE" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +echo "== generating ciphertexts (ore-rs, throwaway key)" +cargo run --locked --quiet --release --manifest-path "$here/ctgen/Cargo.toml" -- 256 >"$tmp/ct.tsv" + +psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 -c "CREATE SCHEMA ore_v2_timing" >/dev/null +created=1 +psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 \ + -c "CREATE TABLE ore_v2_timing.staging (kind text, label text, a_hex text, b_hex text)" \ + -c "\\copy ore_v2_timing.staging FROM '$tmp/ct.tsv'" >/dev/null +psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 -f "$here/harness.sql" >/dev/null + +echo "== correctness (must be 0 wrong before timing means anything)" +wrong="$(psql "$DATABASE_URL" -tA -c " + SELECT count(*) FILTER (WHERE eql_v4_internal.compare_ore_block_v2_terms(a, b) IS DISTINCT FROM expected) + + count(*) FILTER (WHERE eql_v4_internal.compare_ore_block_v2_terms(b, a) IS DISTINCT FROM -expected) + FROM ore_v2_timing.checks")" +echo " wrong: $wrong" +[ "$wrong" = "0" ] || { echo "comparator disagrees with plaintext order; not timing"; exit 1; } + +echo "== timing (two runs per kind; classes: see README.md)" +for run in 1 2; do + psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 -c " + CREATE TABLE ore_v2_timing.fixed_$run AS SELECT * FROM ore_v2_timing.bench('fixed', $samples, 16); + CREATE TABLE ore_v2_timing.equal_$run AS SELECT * FROM ore_v2_timing.bench('equal', $samples, 16); + CREATE TABLE ore_v2_timing.chained_$run AS SELECT * FROM ore_v2_timing.bench('chained', $samples, 8); + CREATE TABLE ore_v2_timing.prefix_$run AS SELECT * FROM ore_v2_timing.bench('prefix', $samples, 8);" +done +psql "$DATABASE_URL" -c " + SELECT * FROM ore_v2_timing.welch('fixed, first vs last block, 16/sample #1', 'ore_v2_timing.fixed_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('fixed, first vs last block, 16/sample #2', 'ore_v2_timing.fixed_2') + UNION ALL SELECT * FROM ore_v2_timing.welch('fixed, equal vs first block, 16/sample #1', 'ore_v2_timing.equal_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('fixed, equal vs first block, 16/sample #2', 'ore_v2_timing.equal_2') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, first vs last block, 8/sample #1', 'ore_v2_timing.chained_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, first vs last block, 8/sample #2', 'ore_v2_timing.chained_2') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, prefix vs first block, 8/sample #1', 'ore_v2_timing.prefix_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, prefix vs first block, 8/sample #2', 'ore_v2_timing.prefix_2');" diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 9040b6e70..0be913577 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -9,6 +9,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A JSON index, `target::Json`** (`docs/plans/2026-10-17-json-tree-index.md`, + ADR-0009). An `Index` whose stored form, + `json::JsonDocument`, is one entry per node under one data key per + document: a path selector and a value selector, the node's own + ciphertext, a token chain the server walks from one client-derived + token, group tags that make containment agree with `jsonb @>` across + array elements, and an ordering term on number and string leaves. It is + what EQL v4's `json_tree` family stores. `target::json_document` seals + one, `JsonDocument` is an `EncryptFrom` / `DecryptInto` + target, and it answers `op::Contains`, `op::ContainedBy`, + `op::AtPath(op::Eq)` (a scalar at a path) and the orderings at a path; + `KeysetCipher::json_selector` and `json_subtree_key` derive the + extraction keys, and `JsonNode` and `JsonSubtree` open what the server + returns. `JsonOptions::sealed_root`, on by default, makes the root entry + the whole document so no token for the root exists. A document is at + most 127 levels deep. New `TermError` variants: `JsonPath`, + `JsonTooDeep`, `JsonTooWide`, `JsonSealedRoot`, `JsonLeafNotOrdered` and + `JsonValueNotScalar`. A data plan reads and writes the index (`"json"`, + or `{"json": {"sealed_root": bool}}`) and refuses to run it with + `Reason::JsonIndexPending`. + +- **A block ORE index, `OreBlock`** (ADR-0002 and ADR-0009, amended + 2026-10-19). It derives ore-rs 0.9.0's 6-bit block ORE under its own key + domain, over the same order keys as the CLLW indexes: + `ore_rs::vitaminc::Bit6BlockOre` for targets of at most ten bytes (the + integers from 16 to 64 bits and both floats) and `ChainedBlockOre` for + text, bytes and the 128-bit integers, text padded to a power-of-two bucket + of at least 16 bytes. As for ORE and OPE, a boolean and an 8-bit integer + have no block ORE term. `sem::OreBlockTerm` is the term, the whole ciphertext; it carries + a random nonce, so it compares by the scheme's comparator (`compare`, + `PartialEq`, `PartialOrd`), never by its bytes, and + `OreBlockTerm::from_bytes` checks a stored term's structure. The typed + path reaches it with `.via(OreBlock)` and `target::ore_block()`, the + dynamic path as the index `ore_block`, and `KeysetCipher::ore_block_term` + derives one directly. A query is the full ciphertext too: EQL v4's + comparator reads the right half of the operand a btree index passes + second. Text whose fold is longer than 1023 bytes is refused with + `TermError::OreBlockTooLong`, text whose fold ends in NUL with + `TermError::OreBlockTrailingNul`, before any key is touched on the query + path; `TermError::OreBlock` carries a scheme failure and + `TermBytesError::MalformedOreBlockCiphertext` a malformed stored term. The + CLLW `Ore` and `Ope` indexes are unchanged, byte for byte. + - **Every integer width, 128-bit integers, dates, timestamps and decimals as dynamic values** (`dynamic` feature; #1144, ADR-0002). They are vitaminc leaves in a `ValueFrame`, with no new `FfiValue` variant: @@ -29,6 +72,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 whole integer into a decimal at scale 0). `tests/value_kinds.rs` and its fixture hold the record path, the data query path and the typed path to the same bytes, and the Go SDK reads the fixture too. +- **A target sealed under a context of its own can opt in to queries on + its plan** (#1238). `op::LabelledTarget` says a target type's stored terms + were derived under the context a plan's label names. A typed plan for such + a type (`Plan::value::().encrypt_into::()`) answers the operators its + `op::Answers` impls name, and needs only the indexes `T` declares, not a + context the label converts into: an EQL v4 type, sealed under its column, + is queried through its plan though it cannot be encrypted through it. + A target type answers queries on its plan only when it implements this + as well as `op::Answers`, whatever context it is sealed under; this + crate's term targets implement both. A type sealed under a context no + label names must not implement it, so its plan answers no query. Naming + an index with `.via(..)` still needs a type sealed under the label itself. + `op::QueryToken` carries the context the query's terms were derived under + (`label`, `is_extended`), for a query term that records it, and + `op::Supports::check_label` (defaulted to `Ok`) lets such an index refuse + a context it cannot record when the query is awaited, before any keyset + is resolved or term derived. - **A binding queries by operator** (`dynamic` feature; ADR-0009, ADR-0007 amended 2026-10-16). `dynamic::query_term(keyset, &plan, field, op, input, &resolver)` derives the query term for a field of a data plan, an @@ -118,14 +178,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **The JSON operators are declared.** `op::Contains` and `op::ContainedBy` (`@>` and `<@`) and the accessor `op::AtPath`, `op::AtPath(op::Eq)`, which applies another operator at a JSON path and inherits its fidelity - (`op::TermFor`): `AtPath` and `AtPath` are exact. No index - answers them yet, since this crate has no JSON index, so a query with - them does not compile until one exists. `AtPath` takes a leaf + (`op::TermFor`): `AtPath` and `AtPath` are exact. The JSON index + above answers them. `AtPath` takes a leaf operator (`op::Leaf`: `Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between`, `Matches`), so `AtPath` and a nested `AtPath>` do not compile, and its arity, `op::Pathed`, carries the leaf's (`Pathed` - for `AtPath`, `Pathed` for `AtPath`); it has no - `op::Input` until the JSON index exists (ADR-0009, decision 1). + for `AtPath`, `Pathed` for `AtPath`), and its + `op::Input` is the JSON index's `(path, value)` and `(path, (lo, hi))` + (ADR-0009, decision 1). - **`target_limit` says how many rows a match query should fetch.** `query.target_limit(want, &stats)` is `want` plus the false positives expected at the confidence `stats` names (99% by default), as an @@ -179,6 +239,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking +- **`IndexSpec` has a sixth variant, `Json(JsonOptions)`** (the JSON index + above), so a `match` over it needs an arm for it. +- **`IndexSpec` has a fifth variant, `OreBlock`** (the block ORE index + above). The enum is exhaustive on purpose, since a binding has to be + taught each index, so a `match` over it needs an arm for `OreBlock`. The + dynamic path answers an ordering through ORE, else block ORE, else OPE, + whichever the field declares. +- **A target's query is handed the index that answers its operator** + (`dynamic` feature). `TargetResolver::query` and `record::query` take an + `&IndexSpec` after the type's name and the field respectively; + `query_term` passes the index `OpSpec::answered_by` picks over the type's + indexes. A type whose query carries every term may ignore it. One whose + terms are optional derives that index's term alone, so EQL v4's + `TextSearch` answers `=` and `<` on text too short to make a match term, + and makes one only for `matches`. A resolver adds the parameter; a caller + of `record::query` passes the index it queries through. - **A ciphertext that opens as the wrong type is `Error::OpenedType`, not `Error::Aead`** (#1118). A typed decrypt of a leaf that authenticates but holds another type (an integer opened as `String`, a value too wide for diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 858816166..be631aedb 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -46,10 +46,19 @@ vitaminc-protected = { workspace = true } # control data (statistics, options, a plan) into typed messages. vitaminc-aead-frame = { workspace = true, optional = true, features = ["serde"] } serde = { workspace = true } -# `Descriptor`: the base64 rendering of a non-textual context for ZeroKMS. +# `Descriptor`: the base64 rendering of a non-textual context for ZeroKMS, +# and the JSON index's ciphertext and header fields on the wire. base64ct = { version = "1.7", features = ["alloc"] } +# The JSON index's token chain: SHA-256 is what links a node to its +# children, because core PostgreSQL computes `sha256(bytea)` and can walk +# the chain server-side (`json` module). The same version vitaminc-hmac +# already builds, so nothing new is compiled. +sha2 = "0.11" cllw-ore = { workspace = true } +# Block ORE (`OreBlock`): ore-rs's 6-bit schemes through the same +# vitaminc-ore traits cllw-ore implements. +ore-rs = { workspace = true } # Every error derives `miette::Diagnostic` (a code, and help where a caller # can act) and gives its structured fields as a `serde_json` map through # `ErrorPayload`. Both are already in the graph through stack-kms. diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index d5df44462..fe99b1ab5 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -1,6 +1,7 @@ --- status: proposed date: 2026-10-13 +amended: 2026-10-19 supersedes: ADR-0004 (decision 5) extends: ADR-0003, ADR-0007 --- @@ -41,6 +42,7 @@ Decision 5 of ADR-0004 assumed they were. In general they are not: | --- | --- | --- | | HMAC | the HMAC | the same HMAC | | ORE | left and right ciphertext | the left ciphertext only | +| block ORE (6-bit; amended 2026-10-19) | the whole ore-rs ciphertext, left and right | the whole ciphertext too, under a fresh nonce: EQL v4's comparator reads its first operand's left half and its second operand's right half, and a btree index or `between` may pass the stored value first. Equal values compare equal; the bytes never match | | OPE | the OPE value of the order key | the same value; equal terms mean equal order keys, which is plaintext equality only when the key is lossless (it is for integers, and not for text after the column's declared transform) | | Bloom match | one filter over every n-gram of the value | a filter over the n-grams of the needle: a subset, or bits the stored filter lacks | | Bloom prefix (planned) | an HMAC per edge n-gram | one HMAC | @@ -157,7 +159,11 @@ pub trait Supports>: Index { `query` makes it from the verifier and the terms the index derived for the query's input. `QueryToken` can be made only inside the crate, so only a query chain calls `query`, and no caller can turn a stored term into a query -term by hand. +term by hand. The token also carries the context the terms were derived +under, for a query term that records it, as an EQL query value stores its +column; such an index refuses a context it cannot record in a defaulted +`check_label`, which the chain runs with `check`, before any term exists +(#1238). `QueryTerm` and the two traits that refine it are defined in decision 4. `Query` is not `Term`. An ORE index's `Supports::Query` is the left @@ -506,6 +512,19 @@ text order encoding, the system ADR-0002's fold pinned to Unicode 16, and the ORE and OPE indexes answer the orderings on text over it. `TextOrdOre` and `TextSearchOre` also need the block ORE scheme EQL stores (#1132). +*Amended 2026-10-19:* they have it. A new index, `IndexSpec::OreBlock` +(wire key `ore_block`), derives ore-rs's 6-bit block ORE under its own key +domain: `Bit6BlockOre` for targets of at most ten bytes (the integers to 64 +bits and the floats) and `ChainedBlockOre` for text, bytes and the 128-bit +integers, over the same order keys the CLLW indexes use. It answers the four +orderings exactly. Its query term is the full ciphertext (see the table +above), and text it cannot take is refused before any term exists: text +whose fold is longer than 1023 bytes or ends in NUL. The CLLW `Ore` and +`Ope` indexes are unchanged, byte for byte. EQL's `ob` key is this index +(`eql-codegen`'s `index_key` maps it to `ore_block`, not to `ore`, which is +CLLW ORE and has no EQL domain), so `TextOrdOre`, `TextSearchOre` and the +integer and float `*OrdOre` types are produced. + A `Ranked` term's `target_limit` applies the index's configured recall factor and needs no `Stats`. diff --git a/packages/stack-encrypt/fuzz/Cargo.lock b/packages/stack-encrypt/fuzz/Cargo.lock index e1111571c..d942dea07 100644 --- a/packages/stack-encrypt/fuzz/Cargo.lock +++ b/packages/stack-encrypt/fuzz/Cargo.lock @@ -21,6 +21,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -227,6 +228,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "bumpalo" version = "3.19.1" @@ -239,6 +246,12 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -1355,6 +1368,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1365,6 +1392,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" @@ -1380,6 +1416,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1411,6 +1468,35 @@ dependencies = [ "libc", ] +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.6", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1995,8 +2081,10 @@ dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", + "sha2 0.11.0", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", @@ -2092,6 +2180,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.119" diff --git a/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs b/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs index ad9265db0..402c6e441 100644 --- a/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs +++ b/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs @@ -104,6 +104,7 @@ impl TargetResolver for Fixed { fn query<'a, K: 'static>( &self, _: &str, + _: &stack_encrypt::target::IndexSpec, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index 3a03cc178..491581033 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -79,6 +79,9 @@ mod tests { errors.extend(boxed(variants![ TermError::Prf(_) => TermError::Prf(cause()), TermError::Ore(_) => TermError::Ore(cllw_ore::Error), + TermError::OreBlockTooLong => TermError::OreBlockTooLong, + TermError::OreBlockTrailingNul => TermError::OreBlockTrailingNul, + TermError::OreBlock(_) => TermError::OreBlock(cause()), TermError::InvalidOptions(_) => TermError::InvalidOptions("k out of range"), TermError::EmptyTermText => TermError::EmptyTermText, TermError::NoEqualityTerm => TermError::NoEqualityTerm, @@ -86,6 +89,15 @@ mod tests { TermError::Normalize(vitaminc_normalize::NormalizeError::UnassignedCodePoint) }, TermError::Bytes(_) => TermError::Bytes(TermBytesError::OddMatchTermsLength(3)), + TermError::JsonPath(_) => TermError::JsonPath(crate::json::JsonPathError { + reason: "expected `$`", + at: 0, + }), + TermError::JsonTooDeep => TermError::JsonTooDeep, + TermError::JsonTooWide => TermError::JsonTooWide, + TermError::JsonSealedRoot => TermError::JsonSealedRoot, + TermError::JsonLeafNotOrdered => TermError::JsonLeafNotOrdered, + TermError::JsonValueNotScalar => TermError::JsonValueNotScalar, ])); errors.extend(boxed(variants![ TermBytesError::WrongEqualityTermLength(_) => { @@ -101,6 +113,9 @@ mod tests { TermBytesError::MalformedCllwCiphertext(_) => { TermBytesError::MalformedCllwCiphertext(3) }, + TermBytesError::MalformedOreBlockCiphertext(_) => { + TermBytesError::MalformedOreBlockCiphertext(3) + }, ])); errors.extend(boxed(variants![ LabelError::Empty => LabelError::Empty, @@ -405,6 +420,12 @@ mod tests { Box::new(TermBytesError::MalformedCllwCiphertext(5)), json!({ "len": 5 }), ), + ( + Box::new(TermBytesError::MalformedOreBlockCiphertext(7)), + json!({ "len": 7 }), + ), + (Box::new(TermError::OreBlockTooLong), json!({})), + (Box::new(TermError::OreBlockTrailingNul), json!({})), ( Box::new(TermBytesError::MatchPositionOutOfRange { position: 900, diff --git a/packages/stack-encrypt/src/dynamic/kind.rs b/packages/stack-encrypt/src/dynamic/kind.rs index d127f98b0..752151bb7 100644 --- a/packages/stack-encrypt/src/dynamic/kind.rs +++ b/packages/stack-encrypt/src/dynamic/kind.rs @@ -55,10 +55,10 @@ use crate::target::IndexSpec; /// `vitaminc-normalize` gives an equality domain, which is every integer /// of every width, both floats, a decimal, a date, a timestamp, text and /// bytes, and not a boolean. The rest are this crate's: match over text -/// alone, and ORE and OPE over the integers of 16 bits and wider, both -/// floats, text and bytes. A boolean and an 8-bit integer have no order -/// term: over so small a domain it would reveal which rows are `true`, or -/// each row's rank among 256 values. +/// alone, and ORE, OPE and block ORE over the integers of 16 bits and +/// wider, both floats, text and bytes. A boolean and an 8-bit integer have +/// no order term: over so small a domain it would reveal which rows are +/// `true`, or each row's rank among 256 values. /// A decimal, a date and a timestamp have none yet: their order encodings /// wait for new cllw-ore and ore-rs releases (#1236), and [`not_admitted`] /// names that reason. A composite (`array`, `object`) has no term. Only the @@ -86,7 +86,7 @@ pub fn admits(kind: ValueKind, index: &IndexSpec) -> bool { /// Why `kind` does not admit `index`, or `None` when it does: the reason a /// plan that declares the pair is refused with. /// -/// [`Reason::OrderTermPending`] for an ORE or OPE index on a decimal, date +/// [`Reason::OrderTermPending`] for an order index on a decimal, date /// or timestamp, whose order terms wait for new cllw-ore and ore-rs /// releases (#1236). [`Reason::IndexNotAdmitted`] for every other pair /// [`admits`] refuses. @@ -103,7 +103,10 @@ pub fn not_admitted(kind: ValueKind, index: &IndexSpec) -> Option { if admits(kind, index) { return None; } - let ordering = matches!(index, IndexSpec::Ore | IndexSpec::Ope); + if matches!(index, IndexSpec::Json(_)) { + return Some(Reason::JsonIndexPending); + } + let ordering = matches!(index, IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock); let pending = matches!( kind, ValueKind::Decimal | ValueKind::Date | ValueKind::Timestamp @@ -581,12 +584,13 @@ mod tests { } } - fn indexes() -> [IndexSpec; 4] { + fn indexes() -> [IndexSpec; 5] { [ IndexSpec::Equality, IndexSpec::Match(MatchOptions::default()), IndexSpec::Ore, IndexSpec::Ope, + IndexSpec::OreBlock, ] } @@ -622,6 +626,9 @@ mod tests { assert!(!admits(ValueKind::Bool, &IndexSpec::Ope)); assert!(!admits(ValueKind::Object, &IndexSpec::Ope)); assert!(!admits(ValueKind::Array, &IndexSpec::Ore)); + assert!(admits(ValueKind::Int32, &IndexSpec::OreBlock)); + assert!(admits(ValueKind::String, &IndexSpec::OreBlock)); + assert!(!admits(ValueKind::Bool, &IndexSpec::OreBlock)); } /// Equality is vitaminc's table: a kind admits it exactly when @@ -677,6 +684,7 @@ mod tests { assert!(admits(kind, &IndexSpec::Equality), "{kind} equality"); assert!(admits(kind, &IndexSpec::Ore), "{kind} ore"); assert!(admits(kind, &IndexSpec::Ope), "{kind} ope"); + assert!(admits(kind, &IndexSpec::OreBlock), "{kind} ore_block"); assert!(!admits(kind, &matched), "{kind} match"); } // An 8-bit integer keeps its equality term and, like a boolean, @@ -696,6 +704,7 @@ mod tests { assert!(admits(kind, &IndexSpec::Equality), "{kind} equality"); assert!(!admits(kind, &IndexSpec::Ore), "{kind} ore"); assert!(!admits(kind, &IndexSpec::Ope), "{kind} ope"); + assert!(!admits(kind, &IndexSpec::OreBlock), "{kind} ore_block"); assert!(!admits(kind, &matched), "{kind} match"); } } @@ -709,11 +718,12 @@ mod tests { for index in indexes() { let reason = not_admitted(kind, &index); assert_eq!(reason.is_none(), admits(kind, &index), "{kind} and {index}"); - let pending = matches!(index, IndexSpec::Ore | IndexSpec::Ope) - && matches!( - kind, - ValueKind::Decimal | ValueKind::Date | ValueKind::Timestamp - ); + let pending = + matches!(index, IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock) + && matches!( + kind, + ValueKind::Decimal | ValueKind::Date | ValueKind::Timestamp + ); if pending { assert_eq!(reason, Some(Reason::OrderTermPending), "{kind} and {index}"); } else if reason.is_some() { @@ -737,11 +747,18 @@ mod tests { not_admitted(ValueKind::Date, &IndexSpec::Match(MatchOptions::default())), Some(Reason::IndexNotAdmitted) ); - assert_eq!(not_admitted(ValueKind::Int16, &IndexSpec::Ore), None); assert_eq!( - not_admitted(ValueKind::Int8, &IndexSpec::Ore), - Some(Reason::IndexNotAdmitted) + not_admitted(ValueKind::Timestamp, &IndexSpec::OreBlock), + Some(Reason::OrderTermPending) ); + assert_eq!(not_admitted(ValueKind::Int16, &IndexSpec::Ore), None); + for index in [IndexSpec::Ore, IndexSpec::Ope, IndexSpec::OreBlock] { + assert_eq!( + not_admitted(ValueKind::Int8, &index), + Some(Reason::IndexNotAdmitted), + "an 8-bit integer is refused {index} as a boolean is, not as pending" + ); + } assert!(Reason::OrderTermPending.to_string().contains("#1236")); } diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index 7ccaabf08..203a251d0 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -534,6 +534,11 @@ reasons! { /// releases that encode them the ADR-0002 way (#1236); equality on /// them works today. OrderTermPending => "order_term_pending", "an order index on a decimal, date or timestamp waits for new cllw-ore and ore-rs releases (#1236)"; + /// The field declares the JSON index, which a data plan names but + /// cannot run yet: the engine's typed path produces a JSON + /// document, and the binding's half, a document plaintext and its + /// EQL target, is not written. + JsonIndexPending => "json_index_pending", "the JSON index has no data-plan form yet"; /// Two fields are keyed under one identity. SharedIdentity => "shared_identity", "two fields are keyed under one identity"; /// The plan has no field of the name asked for. diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index bb835c22e..22ee6f526 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -36,15 +36,15 @@ //! declared kind: `eq` through an equality index, `matches` through a match //! index over text, and the orderings (`lt`, `lte`, `gt`, `gte`, //! `between`) through an ORE index over an integer of 16 bits or wider, a -//! float or text, or an OPE index when the field declares no ORE. The typed -//! path's default for an ordering is ORE, and OPE is the typed path's -//! `.via(Ope)`; here a field that declares only OPE is answered through it. -//! Every order term is derived over the value's ADR-0002 order key: a -//! float's ties `-0.0` with `0.0` and sorts every NaN last, and text's is -//! its fold. An ordering over bytes is not answered, as `op::Ordered` does -//! not cover them. Nor is one over an 8-bit integer, which has no order -//! term, or over a decimal, date or timestamp: a plan cannot declare an -//! order index on one yet. +//! float or text, else a block ORE index, else an OPE index. The typed +//! path's default for an ordering is ORE, and block ORE and OPE are the +//! typed path's `.via(OreBlock)` and `.via(Ope)`; here a field that +//! declares only one of them is answered through it. Every order term is +//! derived over the value's ADR-0002 order key: a float's ties `-0.0` with +//! `0.0` and sorts every NaN last, and text's is its fold. An ordering over +//! bytes is not answered, as `op::Ordered` does not cover them. Nor is one +//! over an 8-bit integer, which has no order term, or over a decimal, date +//! or timestamp: a plan cannot declare an order index on one yet. //! //! A field that names an EQL type answers what its //! [`TargetDescriptor::operators`] lists: the operators of a single value @@ -189,6 +189,7 @@ impl OpSpec { return None; } declared(|index| matches!(index, IndexSpec::Ore)) + .or_else(|| declared(|index| matches!(index, IndexSpec::OreBlock))) .or_else(|| declared(|index| matches!(index, IndexSpec::Ope))) } } @@ -612,6 +613,15 @@ fn check_input(index: &IndexSpec, input: &Input) -> Result<(), Error> { order_text(lo).map_err(refused)?; order_text(hi).map_err(refused)?; } + // Block ORE refuses more order text: a fold longer than 1023 + // bytes, or one that ends in NUL. + (IndexSpec::OreBlock, Input::One(value)) => { + ore_block_text(value).map_err(refused)?; + } + (IndexSpec::OreBlock, Input::Pair(lo, hi)) => { + ore_block_text(lo).map_err(refused)?; + ore_block_text(hi).map_err(refused)?; + } (IndexSpec::Match(options), Input::One(value)) => { query_tokens(value, options).map(drop)?; } @@ -643,6 +653,22 @@ fn order_text(value: &ValueFrame) -> Result<(), TermError> { .unwrap_or(Ok(())) } +/// Whether `value`, if it is order text, has a block ORE term: its fold +/// exists, is at most 1023 bytes and does not end in NUL +/// ([`crate::sem::ore_block_text`]). Anything but text has nothing to +/// refuse here. +fn ore_block_text(value: &ValueFrame) -> Result<(), TermError> { + value + .visit(OneScalar::new( + |ty: Option, value: Physical<'_>| match (ty, value) { + (None, Physical::Text(text)) => Ok(crate::sem::ore_block_text(text)), + _ => Err(None), + }, + )) + // Not one text scalar: nothing to refuse here. + .unwrap_or(Ok(())) +} + /// The tokens a match query of `value` keeps, under the index's options. /// /// # Errors @@ -767,8 +793,12 @@ where return Err(Error::Internal); }; let domain = target.query_sql_domain; - return Ok(super::record::query(cipher, plan, field, value, resolver)? - .map(move |json| made(domain, Operand::One(json)))); + // The answering index goes to the resolver: a type whose query + // carries only its operator's term derives that index's alone. + return Ok( + super::record::query(cipher, plan, field, &answer.index, value, resolver)? + .map(move |json| made(domain, Operand::One(json))), + ); } let context = answer.field.context(); let term = |value: &ValueFrame| { @@ -1479,6 +1509,59 @@ mod tests { assert_eq!(negative_zero.operand, zero.operand); } + /// Block ORE takes order text whose fold is at most 1023 bytes and does + /// not end in NUL. A query is refused for any other, either bound of a + /// range included, before any term exists, by the key-free check too. + #[tokio::test] + async fn block_ore_refuses_text_it_has_no_term_for_before_any_term() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = record::plan(&frame_of(&Tree::Object(vec![field( + "title", + "string", + vec![text("c"), text("ore_block")], + )]))) + .expect("the plan parses"); + let one = |text: &str| ValueFrame::of(text).expect("a value"); + let two = |lo: &str, hi: &str| ValueFrame::of(vec![lo, hi]).expect("a value"); + let longest = "a".repeat(1023); + let too_long = "a".repeat(1024); + let refused = [ + (OpSpec::Lt, one(&too_long), "long"), + (OpSpec::Gt, one("a\0"), "nul"), + (OpSpec::Between, two("a", &too_long), "long"), + (OpSpec::Between, two(&too_long, "b"), "long"), + (OpSpec::Between, two("a\0", "b"), "nul"), + (OpSpec::Between, two("a", "b\0"), "nul"), + ]; + for (op, input, what) in refused { + let derived = match query_term(&keyset, &plan, "title", op, &input, &NoTargets) { + Ok(pending) => pending.await.map_err(Error::Cipher).map(|_| ()), + Err(error) => Err(error), + }; + for error in [ + derived.unwrap_err(), + check_query(&plan, "title", op, &input, &NoTargets).unwrap_err(), + ] { + let term = term_error(&error); + let expected = match what { + "long" => matches!(term, Some(TermError::OreBlockTooLong)), + _ => matches!(term, Some(TermError::OreBlockTrailingNul)), + }; + assert!(expected, "{op:?} {what}: {error:?}"); + } + } + for input in [one(&longest), one("a\0b"), one("")] { + check_query(&plan, "title", OpSpec::Lt, &input, &NoTargets) + .expect("block ORE takes it"); + let term = query_term(&keyset, &plan, "title", OpSpec::Lt, &input, &NoTargets) + .expect("a query") + .await + .expect("a term"); + assert_eq!(term.index, IndexSpec::OreBlock); + } + } + /// A plan whose context is a field of each record has no context a /// query could derive under; the binding sends the plan under the /// context it names. @@ -2362,6 +2445,7 @@ mod tests { fn query<'a, K: 'static>( &self, _: &str, + _: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, @@ -2538,6 +2622,7 @@ mod tests { fn query<'a, K: 'static>( &self, _: &str, + _: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 262e1c53c..903dbbc2c 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -145,7 +145,7 @@ use std::fmt; use serde::de::{Deserializer, MapAccess, Visitor}; use serde::Deserialize; -use super::term::{IndexWire, MatchWire}; +use super::term::{IndexWire, JsonWire, MatchWire}; use super::{ValueFrame, ValueKind}; use vitaminc_aead::frame::{Frame, FrameWriter}; use vitaminc_aead::plaintext::{Logical, MapSink, Physical, SeqSink, Sink}; @@ -1218,6 +1218,9 @@ enum OutputWire { Match(Option), Ore, Ope, + #[serde(rename = "ore_block")] + OreBlock, + Json(Option), } impl From for Output { @@ -1229,6 +1232,8 @@ impl From for Output { OutputWire::Match(options) => IndexWire::Match(options), OutputWire::Ore => IndexWire::Ore, OutputWire::Ope => IndexWire::Ope, + OutputWire::OreBlock => IndexWire::OreBlock, + OutputWire::Json(options) => IndexWire::Json(options), }; Output::Term(index.into()) } @@ -1447,6 +1452,12 @@ pub fn encrypt_with<'a, K: 'static>( /// settles without I/O; it is a [`Pending`] all the same, for one shape at /// the call site. /// +/// `index` is the type's index that answers the query's operator, handed to +/// the resolver ([`TargetResolver::query`]): a type whose query carries only +/// the term its operator reads derives that index's term alone. +/// [`query_term`](super::query_term()) passes the index +/// [`OpSpec::answered_by`](super::OpSpec::answered_by) picks. +/// /// # Errors /// /// [`Error::Plan`] if the plan has no field `field` or it is not a target @@ -1457,6 +1468,7 @@ pub fn query<'a, K: 'static>( cipher: &'a KeysetCipher<'_, K>, plan: &Plan, field: &str, + index: &IndexSpec, value: impl Encrypt, resolver: &(impl TargetResolver + ?Sized), ) -> Result, K>, Error> { @@ -1478,7 +1490,7 @@ pub fn query<'a, K: 'static>( })?; check_field(&value, field)?; resolver - .query(name, cipher, field.label(), value) + .query(name, index, cipher, field.label(), value) .map_err(|error| Error::Target(name_target(error, &field.name))) } @@ -5533,6 +5545,7 @@ mod tests { fn query<'a, K: 'static>( &self, name: &str, + _: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, @@ -6249,16 +6262,30 @@ mod tests { let cipher = cipher().await; let keyset = cipher.default_keyset(); let plan = mixed_plan(); - let probe = query(&keyset, &plan, "email", s("a@x"), &FakeEql) - .expect("a target field") - .await - .expect("derives"); + let probe = query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + s("a@x"), + &FakeEql, + ) + .expect("a target field") + .await + .expect("derives"); let probe: serde_json::Value = serde_json::from_slice(&probe).unwrap(); assert_eq!(probe["i"], "users/email"); - let again = query(&keyset, &plan, "email", s("a@x"), &FakeEql) - .unwrap() - .await - .unwrap(); + let again = query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + s("a@x"), + &FakeEql, + ) + .unwrap() + .await + .unwrap(); assert_eq!( serde_json::from_slice::(&again).unwrap()["hm"], probe["hm"], @@ -6269,17 +6296,38 @@ mod tests { // bare build: each refused before the resolver runs. The first // two name the field asked for and why. for (field, reason) in [("age", Reason::NotATarget), ("nope", Reason::NoSuchField)] { - let error = refused(query(&keyset, &plan, field, s("x"), &FakeEql)); + let error = refused(query( + &keyset, + &plan, + field, + &IndexSpec::Equality, + s("x"), + &FakeEql, + )); assert!(matches!(error, Error::Plan { .. }), "{error:?}"); assert_eq!(error.field(), Some(field), "{error}"); assert_eq!(error.reason(), Some(reason), "{error}"); } assert!(matches!( - refused(query(&keyset, &plan, "email", Tree::UInt32(1), &FakeEql)), + refused(query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + Tree::UInt32(1), + &FakeEql + )), Error::Source { .. } )); assert!(matches!( - refused(query(&keyset, &plan, "email", s("x"), &NoTargets)), + refused(query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + s("x"), + &NoTargets + )), Error::Target(TargetError::NoTargets { .. }) )); } diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index a15a57faf..e34902c2d 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -31,6 +31,7 @@ use super::query::target_operators; use super::{OpSpec, ValueFrame, ValueKind}; use crate::sem::MatchOptions; +use crate::target::IndexSpec; use crate::{KeysetCipher, Label, Pending, StackCipher}; /// One EQL type a plan may name as a target, as the host describes it. @@ -518,12 +519,23 @@ pub trait TargetResolver { /// Run the named type's query twin over one value: the operand that /// matches stored values under `label`'s column, as JSON bytes. /// + /// `index` is the type's index that answers the query's operator, one of + /// the descriptor's [`indexes`](TargetDescriptor::indexes): + /// [`query_term`](super::query_term()) passes the one + /// [`OpSpec::answered_by`] picks. A twin that carries every term of the + /// type may ignore it. A twin whose terms are optional (EQL v4's + /// `TextSearchQuery`) carries that index's term alone, so a query whose + /// operator needs one term is not refused for want of another: a match + /// term, which text shorter than one token cannot make, is derived only + /// for `matches`. + /// /// # Errors /// /// As [`encrypt`](Self::encrypt). fn query<'a, K: 'static>( &self, name: &str, + index: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, @@ -573,6 +585,7 @@ impl TargetResolver for NoTargets { fn query<'a, K: 'static>( &self, name: &str, + _: &IndexSpec, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, @@ -650,6 +663,7 @@ mod tests { fn query<'a, K: 'static>( &self, _: &str, + _: &IndexSpec, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index 1bcb28b7b..7df0ed7e9 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -15,12 +15,13 @@ //! compared in Unicode 16 NFC, and a date, timestamp, decimal or uuid has //! a domain of its own. A boolean has none. //! - **Match** takes text only, cut in Unicode 16 NFC. -//! - **ORE and OPE** take an integer of 16 bits or wider, a float, text or -//! bytes, each handed to cllw-ore as the Rust type the typed path would -//! have named: an `i16` as an `i16`, a `u128` as a `u128`. A boolean, a -//! `u8` and an `i8` have no order term: over a domain that small (two -//! values, or 256) it would tell anyone holding the column which rows are -//! `true`, or rank every row among 256. A decimal, a date and a timestamp +//! - **ORE, OPE and block ORE** take an integer of 16 bits or wider, a +//! float, text or bytes, each handed to cllw-ore (ORE, OPE) or ore-rs +//! (block ORE) as the Rust type the typed path would have named: an `i16` +//! as an `i16`, a `u128` as a `u128`. A boolean, a `u8` and an `i8` have +//! no order term: over a domain that small (two values, or 256) it would +//! tell anyone holding the column which rows are `true`, or rank every +//! row among 256. A decimal, a date and a timestamp //! have none yet: cllw-ore 0.5 does not encode them the ADR-0002 way, and //! their order terms wait for new cllw-ore and ore-rs releases (#1236). //! @@ -44,6 +45,7 @@ use vitaminc_normalize::{domain_of, normalize, NormalizeError}; use serde::{Deserialize, Serialize, Serializer}; use super::{Error, Reason}; +use crate::json::JsonOptions; use crate::sem::{DefaultMatch, MatchOptions, Orderable, QueryTokens, TermError, Tokenizer}; use crate::target::{chosen, CallerContext, ConsumeSource, Encryption, Index, IndexSpec, Pending}; use crate::{Encrypt, IntoPrfContext, KeysetCipher, NonEmpty}; @@ -53,7 +55,8 @@ use crate::{Encrypt, IntoPrfContext, KeysetCipher, NonEmpty}; /// /// # Wire form /// -/// An index is its key, a string: `"eq"`, `"match"`, `"ore"` or `"ope"`. +/// An index is its key, a string: `"eq"`, `"match"`, `"ore"`, `"ope"` or +/// `"ore_block"`. /// `"match"` is a match index under the default options /// ([`MatchOptions::default`]), so every plan written before options had a /// wire form reads as it always did. @@ -69,13 +72,24 @@ use crate::{Encrypt, IntoPrfContext, KeysetCipher, NonEmpty}; /// Every option is optional and defaults to [`MatchOptions::default`]'s /// value; an unknown option, an option given twice, or options that fail /// the match scheme's bounds (`k` in `3..=16`, `m` a power of two in -/// `[32, 65536]`, a non-zero n-gram length) are refused. The other three +/// `[32, 65536]`, a non-zero n-gram length) are refused. +/// +/// A JSON index is `"json"`, or an object mapping `"json"` to its options: +/// +/// ```text +/// { "json": { "sealed_root": } } +/// ``` +/// +/// `sealed_root` defaults to `true` ([`JsonOptions::default`]). A data plan +/// reads and writes the JSON index and refuses to run it +/// ([`Reason::JsonIndexPending`]): the engine's typed path produces a JSON +/// document, and the binding's half is not written yet. The other four /// indexes have no options and no object form. /// /// Serialized (it is `Serialize`), an index writes the string whenever the -/// options are the defaults and the object, with all four options, -/// otherwise, so a default plan is byte-for-byte what it was before options -/// had a wire form. +/// options are the defaults and the object, with every option, otherwise, +/// so a default plan is byte-for-byte what it was before options had a +/// wire form. impl IndexSpec { /// The index a bare key names, under default options, or `None` for a /// string that is not an index key. @@ -85,6 +99,8 @@ impl IndexSpec { "match" => IndexSpec::Match(MatchOptions::default()), "ore" => IndexSpec::Ore, "ope" => IndexSpec::Ope, + "ore_block" => IndexSpec::OreBlock, + "json" => IndexSpec::Json(JsonOptions::default()), _ => return None, }) } @@ -109,9 +125,10 @@ impl IndexSpec { /// is one scalar the index takes. Equality takes every scalar vitaminc /// gives an equality domain (an integer, a float, text, bytes, and a /// date, timestamp, decimal or uuid), not a boolean or a null. Match - /// takes text. ORE and OPE take an integer of any width, a float, text - /// or bytes; not a boolean, and not yet a decimal, date or timestamp - /// (#1236). A match index's options do not change what it takes. + /// takes text. ORE, OPE and block ORE take an integer of any width, a + /// float, text or bytes; not a boolean, and not yet a decimal, date or + /// timestamp (#1236). A match index's options do not change what it + /// takes. /// /// This is the one table: [`term`] refuses exactly what it refuses, and /// checking it first is how a binding refuses a bad request at its @@ -148,18 +165,21 @@ impl IndexSpec { // vitaminc's table: a kind with an equality domain. IndexSpec::Equality => domain_of(ty, value).is_some(), IndexSpec::Match(_) => ty.is_none() && matches!(value, Physical::Text(_)), - IndexSpec::Ore | IndexSpec::Ope => ty.is_none() && ordered(value), + IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock => ty.is_none() && ordered(value), + // A document, not a scalar: the dynamic path has no term for it + // yet, and the typed path takes a `serde_json::Value`. + IndexSpec::Json(_) => false, } } } -/// Whether CLLW ORE and OPE order this scalar: an integer of 16 bits or -/// wider, a float, text or bytes. Not a boolean, and not an 8-bit integer: -/// an order term over so small a domain reveals too much, which rows hold -/// `true`, or each row's rank among 256 values, which a few known rows -/// resolve to the values. Not a null. The caller has already refused a -/// scalar under a logical type: a decimal, date or timestamp's order term -/// waits for a later release. +/// Whether the order indexes (CLLW ORE and OPE, and block ORE) order this +/// scalar: an integer of 16 bits or wider, a float, text or bytes. Not a +/// boolean, and not an 8-bit integer: an order term over so small a domain +/// reveals too much, which rows hold `true`, or each row's rank among 256 +/// values, which a few known rows resolve to the values. Not a null. The +/// caller has already refused a scalar under a logical type: a decimal, +/// date or timestamp's order term waits for a later release. fn ordered(value: &Physical<'_>) -> bool { match value { Physical::Int(Int::I8(_) | Int::U8(_)) | Physical::Null | Physical::Bool(_) => false, @@ -181,6 +201,15 @@ impl Serialize for IndexSpec { Match(MatchOut), Ore, Ope, + #[serde(rename = "ore_block")] + OreBlock, + #[serde(rename = "json")] + DefaultJson, + Json(JsonOut), + } + #[derive(Serialize)] + struct JsonOut { + sealed_root: bool, } #[derive(Serialize)] struct MatchOut { @@ -209,15 +238,21 @@ impl Serialize for IndexSpec { }), IndexSpec::Ore => Out::Ore, IndexSpec::Ope => Out::Ope, + IndexSpec::OreBlock => Out::OreBlock, + IndexSpec::Json(options) if *options == JsonOptions::default() => Out::DefaultJson, + IndexSpec::Json(options) => Out::Json(JsonOut { + sealed_root: options.sealed_root, + }), } .serialize(s) } } /// An index in its wire form, as serde reads it from a frame (see -/// [`IndexSpec`'s docs](IndexSpec#wire-form)): a key, or for a match index a -/// one-entry object mapping `"match"` to its options. A bare `"match"`, and -/// `{"match": null}`, are the default options. +/// [`IndexSpec`'s docs](IndexSpec#wire-form)): a key, or for a match or +/// JSON index a one-entry object mapping its key to its options. A bare +/// `"match"`, and `{"match": null}`, are the default match options, and +/// likewise `"json"`. #[derive(Deserialize)] #[serde(rename_all = "lowercase")] pub(super) enum IndexWire { @@ -225,6 +260,9 @@ pub(super) enum IndexWire { Match(Option), Ore, Ope, + #[serde(rename = "ore_block")] + OreBlock, + Json(Option), } impl From for IndexSpec { @@ -234,10 +272,39 @@ impl From for IndexSpec { IndexWire::Match(options) => IndexSpec::Match(options.map(|o| o.0).unwrap_or_default()), IndexWire::Ore => IndexSpec::Ore, IndexWire::Ope => IndexSpec::Ope, + IndexWire::OreBlock => IndexSpec::OreBlock, + IndexWire::Json(options) => IndexSpec::Json(options.map(|o| o.0).unwrap_or_default()), } } } +/// A JSON index's options object: each option optional, defaulting to +/// [`JsonOptions::default`]'s. +#[derive(Deserialize)] +#[serde(from = "JsonOptionsWire")] +pub(super) struct JsonWire(pub(super) JsonOptions); + +#[derive(Deserialize)] +#[serde(deny_unknown_fields, default)] +struct JsonOptionsWire { + sealed_root: bool, +} + +impl Default for JsonOptionsWire { + fn default() -> Self { + let JsonOptions { sealed_root } = JsonOptions::default(); + Self { sealed_root } + } +} + +impl From for JsonWire { + fn from(wire: JsonOptionsWire) -> Self { + JsonWire(JsonOptions { + sealed_root: wire.sealed_root, + }) + } +} + /// A match index's options object: each option optional, defaulting to /// [`MatchOptions::default`]'s, and the whole checked against the match /// scheme's bounds. @@ -478,12 +545,23 @@ where .match_terms_under::(text, context, options.clone()) .map(|terms| TermBytes(terms.to_bytes()))) } - IndexSpec::Ore => ore_of(cipher, value, context).ok_or(None), - IndexSpec::Ope => ope_of(cipher, value, context).ok_or(None), + IndexSpec::Ore => order_of(cipher, OrderIndex::Ore, value, context).ok_or(None), + IndexSpec::Ope => order_of(cipher, OrderIndex::Ope, value, context).ok_or(None), + IndexSpec::OreBlock => order_of(cipher, OrderIndex::OreBlock, value, context).ok_or(None), + // `takes` refused it above: no scalar is a document. + IndexSpec::Json(_) => Err(None), } } -/// [`IndexSpec::Ore`] per scalar, each handed to the CLLW encryptor as the +/// The order index a scalar's term is derived under. +#[derive(Clone, Copy)] +enum OrderIndex { + Ore, + Ope, + OreBlock, +} + +/// An order index's term per scalar, each handed to the encryptor as the /// Rust type the typed path would have named; `None` for a scalar it does /// not order ([`ordered`]). /// @@ -497,34 +575,9 @@ where /// Each scalar is encoded at the target of the Rust type named here /// ([`Orderable`]), as on the typed path: text by the system ADR-0002's /// fold, so `café` precomposed or with a combining accent, and `Café`, tie. -fn ore_of<'a, 'c, K: 'static, D>( - cipher: &'a KeysetCipher<'_, K>, - value: Physical<'_>, - context: NonEmpty, -) -> Option> -where - D: IntoPrfContext<'c>, -{ - Some(match value { - Physical::Int(Int::I16(v)) => ore(cipher, v, context), - Physical::Int(Int::I32(v)) => ore(cipher, v, context), - Physical::Int(Int::I64(v)) => ore(cipher, v, context), - Physical::Int(Int::I128(v)) => ore(cipher, v, context), - Physical::Int(Int::U16(v)) => ore(cipher, v, context), - Physical::Int(Int::U32(v)) => ore(cipher, v, context), - Physical::Int(Int::U64(v)) => ore(cipher, v, context), - Physical::Int(Int::U128(v)) => ore(cipher, v, context), - Physical::Float(Float::F32(v)) => ore(cipher, v, context), - Physical::Float(Float::F64(v)) => ore(cipher, v, context), - Physical::Text(t) => ore(cipher, t.to_owned(), context), - Physical::Bytes(b) => ore(cipher, b.to_vec(), context), - _ => return None, - }) -} - -/// [`IndexSpec::Ope`] per scalar; see [`ore_of`]. -fn ope_of<'a, 'c, K: 'static, D>( +fn order_of<'a, 'c, K: 'static, D>( cipher: &'a KeysetCipher<'_, K>, + index: OrderIndex, value: Physical<'_>, context: NonEmpty, ) -> Option> @@ -532,40 +585,27 @@ where D: IntoPrfContext<'c>, { Some(match value { - Physical::Int(Int::I16(v)) => ope(cipher, v, context), - Physical::Int(Int::I32(v)) => ope(cipher, v, context), - Physical::Int(Int::I64(v)) => ope(cipher, v, context), - Physical::Int(Int::I128(v)) => ope(cipher, v, context), - Physical::Int(Int::U16(v)) => ope(cipher, v, context), - Physical::Int(Int::U32(v)) => ope(cipher, v, context), - Physical::Int(Int::U64(v)) => ope(cipher, v, context), - Physical::Int(Int::U128(v)) => ope(cipher, v, context), - Physical::Float(Float::F32(v)) => ope(cipher, v, context), - Physical::Float(Float::F64(v)) => ope(cipher, v, context), - Physical::Text(t) => ope(cipher, t.to_owned(), context), - Physical::Bytes(b) => ope(cipher, b.to_vec(), context), + Physical::Int(Int::I16(v)) => order(cipher, index, v, context), + Physical::Int(Int::I32(v)) => order(cipher, index, v, context), + Physical::Int(Int::I64(v)) => order(cipher, index, v, context), + Physical::Int(Int::I128(v)) => order(cipher, index, v, context), + Physical::Int(Int::U16(v)) => order(cipher, index, v, context), + Physical::Int(Int::U32(v)) => order(cipher, index, v, context), + Physical::Int(Int::U64(v)) => order(cipher, index, v, context), + Physical::Int(Int::U128(v)) => order(cipher, index, v, context), + Physical::Float(Float::F32(v)) => order(cipher, index, v, context), + Physical::Float(Float::F64(v)) => order(cipher, index, v, context), + Physical::Text(t) => order(cipher, index, t.to_owned(), context), + Physical::Bytes(b) => order(cipher, index, b.to_vec(), context), _ => return None, }) } -/// The term's raw bytes are its frozen encoding. -fn ore<'a, 'c, K: 'static, T, D>( - cipher: &'a KeysetCipher<'_, K>, - value: T, - context: NonEmpty, -) -> Pending<'a, TermBytes, K> -where - T: Orderable, - D: IntoPrfContext<'c>, -{ - cipher - .ore_term(value, context) - .map(|term| TermBytes(term.as_ref().to_vec())) -} - -/// See [`ore`]. -fn ope<'a, 'c, K: 'static, T, D>( +/// The term of `value` under `index`. The term's bytes are its frozen +/// encoding: the raw CLLW ciphertext, or the serialised block ORE one. +fn order<'a, 'c, K: 'static, T, D>( cipher: &'a KeysetCipher<'_, K>, + index: OrderIndex, value: T, context: NonEmpty, ) -> Pending<'a, TermBytes, K> @@ -573,9 +613,17 @@ where T: Orderable, D: IntoPrfContext<'c>, { - cipher - .ope_term(value, context) - .map(|term| TermBytes(term.as_ref().to_vec())) + match index { + OrderIndex::Ore => cipher + .ore_term(value, context) + .map(|term| TermBytes(term.as_ref().to_vec())), + OrderIndex::Ope => cipher + .ope_term(value, context) + .map(|term| TermBytes(term.as_ref().to_vec())), + OrderIndex::OreBlock => cipher + .ore_block_term(value, context) + .map(|term| TermBytes(term.into_bytes())), + } } /// A sink that takes one scalar and hands it to `F`, and refuses anything @@ -1472,6 +1520,7 @@ mod tests { default_match(), IndexSpec::Ore, IndexSpec::Ope, + IndexSpec::OreBlock, ] { let wire = crate::dynamic::test_frame::value_of(&spec); assert!( diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs new file mode 100644 index 000000000..b222c694d --- /dev/null +++ b/packages/stack-encrypt/src/json/derive.rs @@ -0,0 +1,1519 @@ +//! The derivations: flattening a document into its terms, sealing the +//! entries under one data key, and opening them again. +//! +//! One walk ([`walk`]) serves both the index's term operation and the +//! sealing operation, so a query derives exactly the selectors a stored +//! document carries. The walk is depth-first, pre-order, siblings in key +//! order for an object and in position order for an array; every entry's +//! ordinal `j` is its place in that order, and the AAD of its ciphertext +//! binds it there. +use std::collections::BTreeMap; + +use serde_json::{Map, Number, Value}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; +use vitaminc_aead::{Cipher, Context, IntoAad, IntoContext, LocalCipherText}; +use vitaminc_encrypt::{Aes256Cipher, Key as AesKey}; +use vitaminc_hmac::HmacSha256Prf; +use vitaminc_prf::{BlockVisitor, IntoPrfContext, PrfValue}; +use vitaminc_protected::{Controlled, Protected}; + +use super::path::{JsonPath, Segment}; +use super::query::{JsonSelector, JsonSubtreeKey}; +use super::{JsonDocument, JsonEntry, JsonHeader, JsonNode, JsonOptions, JsonSubtree, Tag}; +use crate::sem::TermError; +use crate::target::{AeadContext, CallerContext, Decryption, Pending, Request}; +use crate::{DecryptInto, Descriptor, Error, KeysetCipher, NonEmpty, StackCipher}; + +/// PAE domain of the path selector `s`. +const SELECTOR_DOMAIN: &[u8] = b"stack-encrypt/json/selector/v1"; +/// PAE domain of the value selector `vs`. +const VALUE_DOMAIN: &[u8] = b"stack-encrypt/json/value/v1"; +/// PAE domain of a chain head's token: the root's, or a top-level field's +/// under the sealed root. +const TOKEN_DOMAIN: &[u8] = b"stack-encrypt/json/token/v1"; +/// PAE domain of an entry's AAD. +const ENTRY_AAD_DOMAIN: &[u8] = b"stack-encrypt/json/entry/v1"; +/// PAE domain of a token step: a child's token, from its parent's token and +/// its label. Keyed, so a token does not let its holder test key names. +const STEP_DOMAIN: &[u8] = b"stack-encrypt/json/token-step/v1"; +/// PAE domain of an object member's rank under its parent's token, which +/// orders the members' ordinals. Keyed for the same reason. +const RANK_DOMAIN: &[u8] = b"stack-encrypt/json/rank/v1"; + +/// Deeper than this is refused rather than sealed: the walk recurses, and a +/// document built in memory can nest deeper than any parser admits. +/// +/// The bound is what reading the document back needs, not what the walk +/// could manage. A sealed root's plaintext is the document as JSON, which +/// `serde_json` parses with a nesting limit of 128 containers, so a +/// document whose innermost value sits 128 levels down seals and then never +/// opens. Refusing it here moves that failure from the read, where the +/// value is lost, to the write, where the caller still holds it. The bound +/// is the same under either root option, so the option never changes which +/// documents the index accepts. +pub(crate) const MAX_DEPTH: usize = 127; + +/// The byte that leads a sealed root's plaintext: the whole document, not a +/// node. +const SEALED_ROOT: u8 = 0xff; + +/// A node's kind: what its value selector says it is, and what its +/// ciphertext's first byte records. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Null, + Bool, + Number, + String, + Object, + Array, +} + +impl Kind { + fn of(value: &Value) -> Self { + match value { + Value::Null => Kind::Null, + Value::Bool(_) => Kind::Bool, + Value::Number(_) => Kind::Number, + Value::String(_) => Kind::String, + Value::Object(_) => Kind::Object, + Value::Array(_) => Kind::Array, + } + } + + pub(crate) fn is_container(self) -> bool { + matches!(self, Kind::Object | Kind::Array) + } + + fn byte(self) -> u8 { + match self { + Kind::Null => 0, + Kind::Bool => 1, + Kind::Number => 2, + Kind::String => 3, + Kind::Object => 4, + Kind::Array => 5, + } + } + + fn from_byte(byte: u8) -> Option { + Some(match byte { + 0 => Kind::Null, + 1 => Kind::Bool, + 2 => Kind::Number, + 3 => Kind::String, + 4 => Kind::Object, + 5 => Kind::Array, + _ => return None, + }) + } + + /// The tag the value selector binds: a container contributes its tag + /// alone, so every object has one value selector at a path and every + /// array another. + fn tag(self) -> &'static [u8] { + match self { + Kind::Null => b"NULL", + Kind::Bool => b"BOOL", + Kind::Number => b"NUM", + Kind::String => b"STR", + Kind::Object => b"MAP0{}", + Kind::Array => b"ARRY[]", + } + } +} + +/// A node's label under its parent. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Label { + Root, + Key(String), + Index(u64), +} + +/// One node's one-way terms: what a query is built from, and what sealing +/// adds the ciphertext and chain fields to. +#[derive(Clone, Debug)] +pub(crate) struct TermNode { + /// The node's positional path. + pub(crate) positional: JsonPath, + pub(crate) s: Tag, + pub(crate) vs: Tag, + /// The node's token: every node has one except the sealed root. + pub(crate) token: Option, + /// The leaf ordering term, on a number or string. + pub(crate) op: Option>, + pub(crate) parent: Option, + /// The node's ordinal under its parent, when the parent has a token. + pub(crate) ordinal: Option, + /// The nearest enclosing array element that is a container. + pub(crate) element: Option, + /// Whether this node is an array element that is a container. + pub(crate) container_element: bool, +} + +/// One node's plaintext, as its ciphertext encodes it. +struct NodePlain { + label: Label, + kind: Kind, + scalar: Option, + children: u32, +} + +/// The one-way terms of a document: one term node per node, in the +/// walk's order, with the root first. The term of the +/// [`Json`](struct@crate::target::Json) index, and what every query term is built +/// from. It holds no scalar value and no key, but it does hold the +/// document's shape: each node's positional path, whose key names are +/// plaintext, and each node's chain token, which is a capability over its +/// subtree in every row. `Debug` prints neither, only how many nodes there +/// are, and the terms should not be logged or sent anywhere but into a +/// query. +#[derive(Clone)] +pub struct JsonTerms { + nodes: Vec, + /// For a document whose root is a scalar: the selector pair the same + /// scalar has as an element of a root array, so a containment needle + /// can match a stored `["a"]` as `jsonb @>` does. + alt: Option<(Tag, Tag)>, +} + +/// The key names and the tokens are what the terms must not leak, so the +/// output is the node count alone. +impl std::fmt::Debug for JsonTerms { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JsonTerms") + .field("nodes", &self.nodes.len()) + .finish_non_exhaustive() + } +} + +impl JsonTerms { + /// The path selector of the node at `path`, which names the node + /// positionally, or `None` when the document has no node there. + pub fn selector(&self, path: &JsonPath) -> Option { + self.nodes + .iter() + .find(|n| &n.positional == path) + .map(|n| JsonSelector(n.s)) + } + + /// How many nodes the document has. + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// Whether the document has no nodes. It never has: a document has a + /// root. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + pub(crate) fn nodes(&self) -> &[TermNode] { + &self.nodes + } + + pub(crate) fn alt(&self) -> Option<(Tag, Tag)> { + self.alt + } + + /// The last node of the walk: in a needle built from one path, the + /// leaf the path names. + pub(crate) fn leaf(&self) -> Option<&TermNode> { + self.nodes.last() + } +} + +// ============================================================================= +// Primitives +// ============================================================================= + +fn truncate(block: &[u8]) -> Tag { + let mut tag = [0; 16]; + tag.copy_from_slice(&block[..16]); + tag +} + +/// The first sixteen bytes of the PRF of `input` under `domain` and the +/// column's context. +fn prf16( + prf: &HmacSha256Prf, + domain: &[u8], + context: &Context<'_>, + input: &[u8], +) -> Result { + let block: [u8; 32] = input + .prf_visit_with_context( + prf, + Context::pae(&[domain, context.as_bytes()]), + BlockVisitor, + ) + .into_result() + .map_err(TermError::from_prf)?; + Ok(truncate(&block)) +} + +/// The first sixteen bytes of SHA-256 over the concatenation of `parts`. +/// Plain SHA-256 by design: the design's "Why plain SHA-256 is enough here" +/// records why length extension does not apply. +fn h16(parts: &[&[u8]]) -> Tag { + let mut hasher = Sha256::new(); + for part in parts { + hasher.update(part); + } + truncate(&hasher.finalize()) +} + +/// A child's token: `PRF(json/token-step ‖ column, PAE(t(parent), 0x01, +/// key))` for an object member, `PAE(t(parent), 0x02, LE64(index))` for an +/// array element, the first sixteen bytes. +/// +/// Keyed with the index key (the design's decision 11). An unkeyed hash of +/// the parent's token and the label would let anyone holding a token test a +/// guessed key name against every child token the walk unwraps below it, in +/// every row of the column. The server never derives a child token from a +/// label (it unwraps `w`), so only the client computes this step, and the +/// client holds the key. +fn child_token( + prf: &HmacSha256Prf, + context: &Context<'_>, + parent: &Tag, + label: &Label, +) -> Result { + let framed = match label { + Label::Key(key) => Context::pae(&[parent, &[0x01], key.as_bytes()]), + Label::Index(index) => Context::pae(&[parent, &[0x02], &index.to_le_bytes()]), + Label::Root => return Ok(*parent), + }; + prf16(prf, STEP_DOMAIN, context, framed.as_bytes()) +} + +/// `H("p" ‖ iv ‖ t(parent) ‖ LE32(ordinal))`: position-keyed, so siblings +/// are unlinkable at rest. +pub(super) fn parent_tag(iv: &[u8], parent: &Tag, ordinal: u32) -> Tag { + h16(&[b"p", iv, parent, &ordinal.to_le_bytes()]) +} + +/// `H("w" ‖ iv ‖ t(parent) ‖ c)`: the mask a child's token is wrapped under. +pub(super) fn mask(iv: &[u8], parent: &Tag, c: &[u8]) -> Tag { + h16(&[b"w", iv, parent, c]) +} + +/// `H("g" ‖ iv ‖ t(element))`: an array element's group tag. +fn group_tag(iv: &[u8], token: &Tag) -> Tag { + h16(&[b"g", iv, token]) +} + +/// `PRF(json/rank ‖ column, PAE(t(parent), key))`: what an object's members +/// are ranked by for their ordinals, so an ordinal does not rank key names. +/// Keyed, as [`child_token`] is, so a token's holder cannot rank guessed +/// key names and compare the order with the ordinals it sees. +fn rank( + prf: &HmacSha256Prf, + context: &Context<'_>, + parent: &Tag, + key: &str, +) -> Result { + let framed = Context::pae(&[parent, key.as_bytes()]); + prf16(prf, RANK_DOMAIN, context, framed.as_bytes()) +} + +pub(super) fn xor(a: &Tag, b: &Tag) -> Tag { + let mut out = [0; 16]; + for (i, byte) in out.iter_mut().enumerate() { + *byte = a[i] ^ b[i]; + } + out +} + +/// A number's canonical bytes for its value selector: an integral value, +/// whichever JSON spelling it had, is its `i128`; anything else is its +/// `f64`. So `2` and `2.0` are one value, as `jsonb` compares them, and +/// `-0.0` is `0`. +fn canonical_number(number: &Number) -> Vec { + fn int(i: i128) -> Vec { + let mut out = Vec::with_capacity(17); + out.push(b'i'); + out.extend_from_slice(&i.to_le_bytes()); + out + } + if let Some(i) = number.as_i64() { + return int(i128::from(i)); + } + if let Some(u) = number.as_u64() { + return int(i128::from(u)); + } + let f = number.as_f64().unwrap_or(0.0); + if f.fract() == 0.0 && f.abs() < 1e38 { + return int(f as i128); + } + let mut out = Vec::with_capacity(9); + out.push(b'f'); + out.extend_from_slice(&f.to_le_bytes()); + out +} + +/// A scalar's canonical bytes; a container contributes nothing beyond its +/// kind's tag. +fn canonical(value: &Value) -> Vec { + match value { + Value::Null | Value::Object(_) | Value::Array(_) => Vec::new(), + Value::Bool(b) => vec![u8::from(*b)], + Value::Number(n) => canonical_number(n), + Value::String(s) => s.as_bytes().to_vec(), + } +} + +/// The value selector: the PRF of the kind's tag, the path selector and +/// the canonical value, PAE-framed. +fn value_selector( + prf: &HmacSha256Prf, + context: &Context<'_>, + s: &Tag, + value: &Value, +) -> Result { + let framed = Context::pae(&[Kind::of(value).tag(), s, &canonical(value)]); + prf16(prf, VALUE_DOMAIN, context, framed.as_bytes()) +} + +/// The leaf ordering term: the OPE term of a number as an `f64` or of a +/// string, under the column's context, as the scalar `Ope` index derives +/// it. Nothing for any other kind. An integer wider than 53 bits orders by +/// the `f64` it rounds to. +fn leaf_op( + prf: &HmacSha256Prf, + context: &Context<'_>, + value: &Value, +) -> Result>, TermError> { + Ok(match value { + Value::Number(n) => Some( + crate::sem::ope(prf, n.as_f64().unwrap_or(0.0), context.clone())? + .into_inner() + .as_ref() + .to_vec(), + ), + Value::String(s) => Some( + crate::sem::ope(prf, s.clone(), context.clone())? + .into_inner() + .as_ref() + .to_vec(), + ), + _ => None, + }) +} + +// ============================================================================= +// The walk +// ============================================================================= + +struct Walked { + terms: JsonTerms, + plain: Vec, +} + +struct Walker<'p> { + prf: &'p HmacSha256Prf, + context: &'p Context<'p>, + options: JsonOptions, + nodes: Vec, + plain: Vec, +} + +/// Flatten `value` into its terms and its nodes' plaintexts. +fn walk( + value: &Value, + options: JsonOptions, + prf: &HmacSha256Prf, + context: &Context<'_>, +) -> Result { + let mut walker = Walker { + prf, + context, + options, + nodes: Vec::new(), + plain: Vec::new(), + }; + let _ = walker.visit( + value, + &JsonPath::root(), + &JsonPath::root(), + Label::Root, + None, + None, + None, + None, + 0, + )?; + let alt = if Kind::of(value).is_container() { + None + } else { + let wild = JsonPath::root().then(Segment::Wildcard); + let s = prf16(prf, SELECTOR_DOMAIN, context, &wild.selector_bytes())?; + let vs = value_selector(prf, context, &s, value)?; + Some((s, vs)) + }; + Ok(Walked { + terms: JsonTerms { + nodes: walker.nodes, + alt, + }, + plain: walker.plain, + }) +} + +impl Walker<'_> { + #[allow(clippy::too_many_arguments)] + fn visit( + &mut self, + value: &Value, + positional: &JsonPath, + wild: &JsonPath, + label: Label, + parent: Option, + parent_token: Option, + ordinal: Option, + element: Option, + depth: usize, + ) -> Result { + if depth > MAX_DEPTH { + return Err(TermError::JsonTooDeep); + } + let kind = Kind::of(value); + let s = prf16( + self.prf, + SELECTOR_DOMAIN, + self.context, + &wild.selector_bytes(), + )?; + let vs = value_selector(self.prf, self.context, &s, value)?; + let token = match (depth, self.options.sealed_root) { + (0, true) => None, + (0, false) | (1, true) => Some(prf16( + self.prf, + TOKEN_DOMAIN, + self.context, + &positional.token_bytes().unwrap_or_default(), + )?), + _ => parent_token + .map(|p| child_token(self.prf, self.context, &p, &label)) + .transpose()?, + }; + let op = leaf_op(self.prf, self.context, value)?; + let container_element = matches!(label, Label::Index(_)) && kind.is_container(); + let index = self.nodes.len(); + self.nodes.push(TermNode { + positional: positional.clone(), + s, + vs, + token, + op, + parent, + ordinal, + element, + container_element, + }); + self.plain.push(NodePlain { + label, + kind, + scalar: (!kind.is_container()).then(|| value.clone()), + children: 0, + }); + let child_element = if container_element { + Some(index) + } else { + element + }; + let mut count: u32 = 0; + match value { + Value::Object(map) => { + let ordinals: Option> = token + .map(|t| shuffled_ordinals(self.prf, self.context, &t, map)) + .transpose()?; + for (key, child) in map { + let ordinal = ordinals.as_ref().and_then(|o| o.get(key).copied()); + let _ = self.visit( + child, + &positional.then(Segment::Key(key.clone())), + &wild.then(Segment::Key(key.clone())), + Label::Key(key.clone()), + Some(index), + token, + ordinal, + child_element, + depth + 1, + )?; + count += 1; + } + } + Value::Array(items) => { + for (i, child) in items.iter().enumerate() { + let i = u64::try_from(i).map_err(|_| TermError::JsonTooWide)?; + let ordinal = token + .map(|_| u32::try_from(i)) + .transpose() + .map_err(|_| TermError::JsonTooWide)?; + let _ = self.visit( + child, + &positional.then(Segment::Index(i)), + &wild.then(Segment::Wildcard), + Label::Index(i), + Some(index), + token, + ordinal, + child_element, + depth + 1, + )?; + count += 1; + } + } + _ => {} + } + self.plain[index].children = count; + Ok(index) + } +} + +/// An object's members' ordinals: their positions in the order of their +/// ranks under the parent's token, so the ordinals rank nothing a reader +/// without the token can see. +fn shuffled_ordinals( + prf: &HmacSha256Prf, + context: &Context<'_>, + parent: &Tag, + map: &Map, +) -> Result, TermError> { + let mut ranked: Vec<(Tag, &String)> = map + .keys() + .map(|k| Ok((rank(prf, context, parent, k)?, k))) + .collect::>()?; + ranked.sort(); + Ok(ranked + .into_iter() + .enumerate() + .map(|(o, (_, key))| (key.clone(), o as u32)) + .collect()) +} + +// ============================================================================= +// Sealing +// ============================================================================= + +/// The one-way terms of `value` under `context`, as the index's operation +/// derives them: no key is requested. +pub(crate) fn terms_pending<'a, K>( + value: &Value, + options: JsonOptions, + cipher: &'a KeysetCipher<'_, K>, + context: NonEmpty, +) -> Pending<'a, JsonTerms, K> { + let context = context.into_prf_context().into_owned(); + let terms = walk(value, options, cipher.prf(), &context) + .map(|w| w.terms) + .map_err(Error::from); + Pending::ready(cipher, terms) +} + +/// Seal `value` into a [`JsonDocument`] under `context`: one data key for +/// the document, requested under the context's descriptor, and every entry +/// sealed under it with the AAD that binds the entry to the document, the +/// column and its ordinal. +pub(crate) fn seal_pending<'a, K: 'static>( + value: Value, + options: JsonOptions, + cipher: &'a KeysetCipher<'_, K>, + context: NonEmpty, +) -> Pending<'a, JsonDocument, K> { + let piece = context.into_context(); + let descriptor = Descriptor::from_piece(&piece); + if let Err(error) = descriptor.check() { + return Pending::failed(cipher, error); + } + let prf_context = piece.clone().into_prf_context().into_owned(); + let aad = piece.into_aad().into_owned(); + let walked = match walk(&value, options, cipher.prf(), &prf_context) { + Ok(walked) => walked, + Err(error) => return Pending::failed(cipher, error.into()), + }; + let whole = if options.sealed_root { + match serde_json::to_vec(&value) { + Ok(bytes) => Some(bytes), + Err(error) => return Pending::failed(cipher, Error::Other(Box::new(error))), + } + } else { + None + }; + let keyset_id = cipher.keyset_id(); + Pending::request( + cipher, + vec![Request::generate_under(descriptor)], + move |responses| { + let key = responses.next_generated_key()?; + seal(walked, whole, keyset_id, key, &aad, options) + }, + ) +} + +/// An entry's AAD: the domain, the keyset, the column's AAD, the key tag, +/// the entry's selector and its ordinal. +fn entry_aad(keyset_id: Uuid, aad: &Context<'_>, tag: &[u8], s: &Tag, j: u64) -> Context<'static> { + Context::pae(&[ + ENTRY_AAD_DOMAIN, + keyset_id.as_bytes(), + aad.as_bytes(), + tag, + s, + &j.to_le_bytes(), + ]) +} + +fn leaf_cipher(key: &stack_kms::DataKey) -> Result { + Aes256Cipher::new(&AesKey::from(*key.key())).map_err(|_| Error::Aead) +} + +/// A node's plaintext: its kind, its child count, its label and, for a +/// scalar, its value. +fn encode_node(plain: &NodePlain) -> Vec { + let mut out = vec![plain.kind.byte()]; + out.extend_from_slice(&plain.children.to_le_bytes()); + match &plain.label { + Label::Root => out.push(0x00), + Label::Key(key) => { + out.push(0x01); + out.extend_from_slice(&(key.len() as u32).to_le_bytes()); + out.extend_from_slice(key.as_bytes()); + } + Label::Index(index) => { + out.push(0x02); + out.extend_from_slice(&index.to_le_bytes()); + } + } + match &plain.scalar { + Some(Value::String(s)) => out.extend_from_slice(s.as_bytes()), + Some(Value::Number(n)) => out.extend_from_slice(n.to_string().as_bytes()), + Some(Value::Bool(b)) => out.push(u8::from(*b)), + _ => {} + } + out +} + +fn seal( + walked: Walked, + whole: Option>, + keyset_id: Uuid, + key: stack_kms::DataKeyWithTag, + aad: &Context<'_>, + options: JsonOptions, +) -> Result { + if key.tag.len() > usize::from(u16::MAX) { + return Err(Error::Other( + "the data key's tag does not fit the JSON header".into(), + )); + } + let cipher = leaf_cipher(&key.key)?; + let iv = key.key.iv; + let tag = key.tag; + let nodes = &walked.terms.nodes; + let mut entries = Vec::with_capacity(nodes.len()); + for (j, (node, plain)) in nodes.iter().zip(&walked.plain).enumerate() { + let plaintext = match (&whole, j) { + (Some(whole), 0) => { + let mut out = Vec::with_capacity(whole.len() + 1); + out.push(SEALED_ROOT); + out.extend_from_slice(whole); + out + } + _ => encode_node(plain), + }; + let c = cipher + .seal( + Protected::new(plaintext), + &entry_aad(keyset_id, aad, &tag, &node.s, j as u64), + ) + .map_err(|_| Error::Aead)?; + let c = c.as_ref().to_vec(); + let parent_token = node.parent.and_then(|p| nodes[p].token); + let (o, p, w) = match (parent_token, node.ordinal, node.token) { + (Some(parent), Some(o), Some(token)) => ( + Some(o), + Some(parent_tag(&iv, &parent, o)), + Some(xor(&token, &mask(&iv, &parent, &c))), + ), + _ => (None, None, None), + }; + let n = if node.container_element { + node.token.map(|t| group_tag(&iv, &t)) + } else { + None + }; + let g = node + .element + .and_then(|e| nodes[e].token) + .map(|t| group_tag(&iv, &t)); + entries.push(JsonEntry { + s: node.s, + vs: node.vs, + c, + o, + p, + w, + n, + g, + op: node.op.clone(), + }); + } + Ok(JsonDocument::new( + JsonHeader::new(keyset_id, iv, tag, options.sealed_root), + entries, + )) +} + +// ============================================================================= +// Opening +// ============================================================================= + +/// An opened node. +struct Opened { + label: Label, + kind: Kind, + scalar: Option, + children: u32, + /// The whole document, from a sealed root. + whole: Option, +} + +fn shape(expected: &'static str) -> Error { + Error::OpenedType { expected } +} + +fn decode_node(bytes: &[u8]) -> Result { + let refused = || shape("a JSON index node"); + let (&first, rest) = bytes.split_first().ok_or_else(refused)?; + if first == SEALED_ROOT { + let whole: Value = serde_json::from_slice(rest).map_err(|_| refused())?; + return Ok(Opened { + label: Label::Root, + kind: Kind::of(&whole), + scalar: None, + children: 0, + whole: Some(whole), + }); + } + let kind = Kind::from_byte(first).ok_or_else(refused)?; + if rest.len() < 5 { + return Err(refused()); + } + let (count, rest) = rest.split_at(4); + let children = u32::from_le_bytes([count[0], count[1], count[2], count[3]]); + let (&label_tag, rest) = rest.split_first().ok_or_else(refused)?; + let (label, rest) = match label_tag { + 0x00 => (Label::Root, rest), + 0x01 => { + if rest.len() < 4 { + return Err(refused()); + } + let (len, rest) = rest.split_at(4); + let len = u32::from_le_bytes([len[0], len[1], len[2], len[3]]) as usize; + if rest.len() < len { + return Err(refused()); + } + let (key, rest) = rest.split_at(len); + let key = std::str::from_utf8(key).map_err(|_| refused())?; + (Label::Key(key.to_owned()), rest) + } + 0x02 => { + if rest.len() < 8 { + return Err(refused()); + } + let (index, rest) = rest.split_at(8); + let mut bytes = [0; 8]; + bytes.copy_from_slice(index); + (Label::Index(u64::from_le_bytes(bytes)), rest) + } + _ => return Err(refused()), + }; + let scalar = match kind { + Kind::Null => Some(Value::Null), + Kind::Bool => Some(Value::Bool(rest.first().is_some_and(|&b| b != 0))), + Kind::Number => { + let text = std::str::from_utf8(rest).map_err(|_| refused())?; + Some(Value::Number(text.parse().map_err(|_| refused())?)) + } + Kind::String => Some(Value::String( + std::str::from_utf8(rest).map_err(|_| refused())?.to_owned(), + )), + Kind::Object | Kind::Array => None, + }; + Ok(Opened { + label, + kind, + scalar, + children, + whole: None, + }) +} + +fn open_entry( + cipher: &Aes256Cipher, + header: &JsonHeader, + aad: &Context<'_>, + entry: &JsonEntry, + j: u64, +) -> Result { + let plaintext = cipher + .open( + LocalCipherText::from(entry.c.clone()), + &entry_aad(header.keyset_id, aad, &header.tag, &entry.s, j), + ) + .map_err(|_| Error::Aead)?; + decode_node(&plaintext.risky_unwrap()) +} + +/// Rebuild the value of `node`, taking its descendants from `rest`, the +/// pre-order sequence after it. Every child is taken with `next`, so each +/// step consumes an entry and the rebuild ends on any sequence. +/// +/// `depth` is `node`'s depth below the rebuild's start. Each entry is +/// authentic, but which entries the server returned, and so how deep the +/// containers it lines up nest, is not: rows that repeat one container, or +/// list only the containers of a wide document, would otherwise recurse as +/// deep as there are rows. Nothing the index seals nests deeper than +/// [`MAX_DEPTH`], so deeper is refused as a shape mismatch. +fn rebuild<'a>( + node: &'a Opened, + rest: &mut impl Iterator, + depth: usize, +) -> Result { + if let Some(whole) = &node.whole { + return Ok(whole.clone()); + } + if depth > MAX_DEPTH { + return Err(shape("a JSON document no deeper than the index seals")); + } + match node.kind { + Kind::Object => { + let mut map = Map::new(); + for _ in 0..node.children { + let member = next_child(rest)?; + let Label::Key(key) = &member.label else { + return Err(shape("an object member")); + }; + let value = rebuild(member, rest, depth + 1)?; + let _ = map.insert(key.clone(), value); + } + Ok(Value::Object(map)) + } + Kind::Array => { + let mut items = Vec::with_capacity(node.children as usize); + for i in 0..node.children { + let element = next_child(rest)?; + if element.label != Label::Index(u64::from(i)) { + return Err(shape("an array element in order")); + } + items.push(rebuild(element, rest, depth + 1)?); + } + Ok(Value::Array(items)) + } + _ => node.scalar.clone().ok_or_else(|| shape("a JSON scalar")), + } +} + +fn next_child<'a>(rest: &mut impl Iterator) -> Result<&'a Opened, Error> { + rest.next().ok_or_else(|| shape("a complete JSON subtree")) +} + +/// The retrieve request for a document's one data key, and the AAD its +/// entries open under. +fn retrieval( + header: &JsonHeader, + context: AeadContext, +) -> Result<(Request, Context<'static>), Error> { + let piece = context.into_context(); + let descriptor = Descriptor::from_piece(&piece); + descriptor.check()?; + Ok(( + Request::retrieve_under(header.iv, header.tag.clone(), descriptor, header.keyset_id), + piece.into_aad().into_owned(), + )) +} + +/// Open the document: one key retrieval, then the sealed root alone, or +/// every entry rebuilt into the value. +fn open_document<'a, K>( + doc: JsonDocument, + cipher: &'a StackCipher, + context: AeadContext, +) -> Pending<'a, Value, K> { + let (request, aad) = match retrieval(&doc.header, context) { + Ok(parts) => parts, + Err(error) => return Pending::failed(cipher, error), + }; + Pending::request(cipher, vec![request], move |responses| { + let key = responses.next_retrieved_key()?; + let aes = leaf_cipher(&key)?; + if doc.header.sealed_root { + let root = doc + .entries + .first() + .ok_or_else(|| shape("a document with a root entry"))?; + return open_entry(&aes, &doc.header, &aad, root, 0)? + .whole + .ok_or_else(|| shape("a sealed root")); + } + let opened = doc + .entries + .iter() + .enumerate() + .map(|(j, entry)| open_entry(&aes, &doc.header, &aad, entry, j as u64)) + .collect::, _>>()?; + let mut rest = opened.iter(); + let root = rest + .next() + .ok_or_else(|| shape("a document with a root entry"))?; + let value = rebuild(root, &mut rest, 0)?; + if rest.next().is_some() { + return Err(shape("a document with no entries past its root")); + } + Ok(value) + }) +} + +impl DecryptInto for JsonDocument { + type Context = AeadContext; + fn decryption(self, context: Self::Context) -> Decryption { + Decryption::from_open(move |cipher| open_document(self, cipher, context)) + } +} + +/// Open one extracted node: the scalar it holds, or the whole document for +/// a sealed root. A container node holds no value of its own and is +/// refused with [`Error::OpenedType`]; its subtree opens through +/// [`JsonSubtree`]. +impl DecryptInto for JsonNode { + type Context = AeadContext; + fn decryption(self, context: Self::Context) -> Decryption { + Decryption::from_open(move |cipher| { + let (request, aad) = match retrieval(&self.header, context) { + Ok(parts) => parts, + Err(error) => return Pending::failed(cipher, error), + }; + Pending::request(cipher, vec![request], move |responses| { + let key = responses.next_retrieved_key()?; + let aes = leaf_cipher(&key)?; + let opened = open_entry(&aes, &self.header, &aad, &self.entry, self.j)?; + opened + .whole + .or(opened.scalar) + .ok_or_else(|| shape("a JSON scalar, or a sealed root")) + }) + }) + } +} + +/// Open the rows of a walk: each complete subtree the rows hold, as a +/// value, in order. What `json_tree_subtree_at` returns is one subtree per +/// node whose selector matched; what `json_tree_subtree` returns is the +/// children of one node, which [`DecryptInto`] assembles into that +/// node's value. +impl DecryptInto> for JsonSubtree { + type Context = AeadContext; + fn decryption(self, context: Self::Context) -> Decryption, K> { + Decryption::from_open(move |cipher| { + open_subtree(self, cipher, context) + .map(|forest| forest.into_iter().map(|(_, v)| v).collect()) + }) + } +} + +/// Open the rows of `json_tree_subtree(doc, token)` into the value under +/// that token: an array when the rows are elements, an object when they are +/// members. No rows is no value ([`Error::OpenedType`]): an empty container +/// and a scalar both have no descendants, and `->` on the node's selector +/// returns the node itself. +impl DecryptInto for JsonSubtree { + type Context = AeadContext; + fn decryption(self, context: Self::Context) -> Decryption { + Decryption::from_open(move |cipher| { + open_subtree(self, cipher, context).try_map(|forest| { + let mut members = Map::new(); + let mut elements: Vec<(u64, Value)> = Vec::new(); + for (label, value) in forest { + match label { + Label::Key(key) => { + let _ = members.insert(key, value); + } + Label::Index(i) => elements.push((i, value)), + Label::Root => return Err(shape("a subtree below the root")), + } + } + match (members.is_empty(), elements.is_empty()) { + (false, true) => Ok(Value::Object(members)), + (true, false) => { + elements.sort_by_key(|(i, _)| *i); + Ok(Value::Array(elements.into_iter().map(|(_, v)| v).collect())) + } + (true, true) => Err(shape("a node with children")), + (false, false) => Err(shape("the children of one node")), + } + }) + }) + } +} + +/// Open every row and rebuild each complete subtree, labelled. +fn open_subtree<'a, K>( + subtree: JsonSubtree, + cipher: &'a StackCipher, + context: AeadContext, +) -> Pending<'a, Vec<(Label, Value)>, K> { + let mut nodes = subtree.nodes; + nodes.sort_by_key(|n| n.j); + // A walk returns each entry once. Two rows at one ordinal are the + // server repeating an entry, which would let it line up one container + // as many times as it likes. + if nodes.windows(2).any(|pair| pair[0].j == pair[1].j) { + return Pending::failed(cipher, shape("rows of distinct entries")); + } + let Some(first) = nodes.first() else { + return Pending::ready(cipher, Ok(Vec::new())); + }; + let header = first.header.clone(); + if nodes.iter().any(|n| n.header != header) { + return Pending::failed(cipher, shape("rows of one document")); + } + let (request, aad) = match retrieval(&header, context) { + Ok(parts) => parts, + Err(error) => return Pending::failed(cipher, error), + }; + Pending::request(cipher, vec![request], move |responses| { + let key = responses.next_retrieved_key()?; + let aes = leaf_cipher(&key)?; + let opened = nodes + .iter() + .map(|n| open_entry(&aes, &header, &aad, &n.entry, n.j)) + .collect::, _>>()?; + let mut forest = Vec::new(); + let mut rest = opened.iter(); + while let Some(top) = rest.next() { + let value = rebuild(top, &mut rest, 0)?; + forest.push((top.label.clone(), value)); + } + Ok(forest) + }) +} + +// ============================================================================= +// Extraction keys, on the keyset cipher +// ============================================================================= + +/// The token of the node at `path`, which must be positional: a chain head +/// by PRF, then one hash per step below it. +fn token_for( + prf: &HmacSha256Prf, + context: &Context<'_>, + options: JsonOptions, + path: &JsonPath, +) -> Result { + let segments = path.segments(); + let (head, rest) = if options.sealed_root { + let Some((first, rest)) = segments.split_first() else { + return Err(TermError::JsonSealedRoot); + }; + let head = JsonPath::root().then(first.clone()); + (head, rest) + } else { + (JsonPath::root(), segments) + }; + let mut token = prf16( + prf, + TOKEN_DOMAIN, + context, + &head.token_bytes().ok_or(TermError::JsonSealedRoot)?, + )?; + for segment in rest { + let label = match segment { + Segment::Key(key) => Label::Key(key.clone()), + Segment::Index(index) => Label::Index(*index), + // `anchor` cut the path before any wildcard. + Segment::Wildcard => return Err(TermError::JsonSealedRoot), + }; + token = child_token(prf, context, &token, &label)?; + } + Ok(token) +} + +impl KeysetCipher<'_, K> { + /// The selector of `path`, under the field `descriptor`: what `->` and + /// the path functions take. Every element of an array shares its + /// parent's selector, so a position in `path` reads as the wildcard. + /// Derived locally; the pending carries no request. + pub fn json_selector<'c>( + &self, + path: &JsonPath, + descriptor: NonEmpty>, + ) -> Pending<'_, JsonSelector, K> { + let context = descriptor.into_prf_context().into_owned(); + let selector = prf16( + self.prf(), + SELECTOR_DOMAIN, + &context, + &path.selector_bytes(), + ) + .map(JsonSelector) + .map_err(Error::from); + Pending::ready(self, selector) + } + + /// The key that walks the subtree under `path`, under the field + /// `descriptor` and the index's `options`: the token of the deepest + /// positional node on the path, and, when the path continues with a + /// wildcard below it, the selector of the whole path, which the walk + /// gates on. What `json_tree_subtree` and `json_tree_subtree_at` take. + /// + /// # Errors + /// + /// [`TermError::JsonSealedRoot`] for the root, or for a path whose + /// first step is a wildcard, under the sealed root: no token exists for + /// the root, and the document opens whole instead. + pub fn json_subtree_key<'c>( + &self, + path: &JsonPath, + options: JsonOptions, + descriptor: NonEmpty>, + ) -> Pending<'_, JsonSubtreeKey, K> { + let context = descriptor.into_prf_context().into_owned(); + let (anchor, cut) = path.anchor(); + let key = token_for(self.prf(), &context, options, &anchor).and_then(|token| { + let selector = cut + .then(|| { + prf16( + self.prf(), + SELECTOR_DOMAIN, + &context, + &path.selector_bytes(), + ) + }) + .transpose()?; + Ok(JsonSubtreeKey { token, selector }) + }); + Pending::ready(self, key.map_err(Error::from)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use vitaminc_prf::PrfKeyInit; + + fn prf() -> HmacSha256Prf { + HmacSha256Prf::new(Protected::new([7; 32])) + } + + fn terms(value: &Value, options: JsonOptions) -> JsonTerms { + walk( + value, + options, + &prf(), + &Context::from_encoded(b"users/profile"), + ) + .unwrap() + .terms + } + + #[test] + fn numbers_canonicalise_by_value() { + let n = |text: &str| -> Number { text.parse().unwrap() }; + assert_eq!(canonical_number(&n("2")), canonical_number(&n("2.0"))); + assert_eq!(canonical_number(&n("0")), canonical_number(&n("-0.0"))); + assert_ne!(canonical_number(&n("2")), canonical_number(&n("2.5"))); + assert_ne!(canonical_number(&n("1")), canonical_number(&n("-1"))); + assert_eq!( + canonical_number(&n("18446744073709551615")), + canonical_number(&n("18446744073709551615")) + ); + // A value selector is stored, so where an integral float stops + // reading as an integer is part of the stored form: below 1e38 it + // is the `i128`, from 1e38 up the `f64`. + assert_eq!(canonical_number(&n("9.9e37"))[0], b'i'); + assert_eq!(canonical_number(&n("1e38"))[0], b'f'); + assert_eq!(canonical_number(&n("-1e38"))[0], b'f'); + } + + #[test] + fn the_terms_name_a_node_by_its_path() { + let t = terms(&json!({"a": [1, {"b": 2}]}), JsonOptions::default()); + // $, $.a, $.a[0], $.a[1], $.a[1].b + assert_eq!(t.len(), 5); + assert!(!t.is_empty()); + let at = |path: &str| t.selector(&JsonPath::parse(path).unwrap()); + assert_eq!(at("$.a[1].b"), Some(JsonSelector(t.nodes()[4].s))); + assert_eq!(at("$.a"), Some(JsonSelector(t.nodes()[1].s))); + assert_eq!(at("$.a[0]"), at("$.a[1]"), "every element shares one"); + assert_eq!(at("$.c"), None, "no node there"); + assert_eq!(at("$.a[2]"), None, "no third element"); + } + + #[test] + fn the_terms_print_neither_key_names_nor_tokens() { + let t = terms( + &json!({"alice@example.com": {"ssn": "123"}}), + JsonOptions::default(), + ); + let printed = format!("{t:?}"); + assert_eq!(printed, "JsonTerms { nodes: 3, .. }"); + let token = t.nodes()[1].token.unwrap(); + assert!(!printed.contains("alice")); + assert!(!printed.contains(&super::super::hex(&token))); + } + + #[test] + fn every_element_of_an_array_shares_a_selector_and_values_do_not() { + let t = terms(&json!({"a": [1, 2, 1]}), JsonOptions::default()); + let nodes = t.nodes(); + assert_eq!(nodes.len(), 5); + assert_eq!(nodes[2].s, nodes[3].s); + assert_eq!(nodes[2].s, nodes[4].s); + assert_eq!(nodes[2].vs, nodes[4].vs, "equal values at one path"); + assert_ne!(nodes[2].vs, nodes[3].vs); + } + + #[test] + fn the_sealed_root_has_no_token_and_its_fields_are_heads() { + let sealed = terms(&json!({"a": {"b": 1}}), JsonOptions::default()); + assert!(sealed.nodes()[0].token.is_none()); + assert!(sealed.nodes()[1].token.is_some()); + assert!(sealed.nodes()[1].ordinal.is_none(), "a head has no ordinal"); + assert!(sealed.nodes()[2].ordinal.is_some()); + let open = terms(&json!({"a": {"b": 1}}), JsonOptions { sealed_root: false }); + assert!(open.nodes()[0].token.is_some()); + assert!(open.nodes()[1].ordinal.is_some()); + // The chain below a head is the same hash chain either way, but + // the heads differ: one is a PRF of the path, the other a hash + // under the root's token. + assert_ne!(sealed.nodes()[1].token, open.nodes()[1].token); + } + + #[test] + fn a_head_token_is_what_the_subtree_key_derives() { + let context = Context::from_encoded(b"users/profile"); + let value = json!({"a": {"b": [1, {"c": 2}]}}); + for options in [JsonOptions::default(), JsonOptions { sealed_root: false }] { + let t = terms(&value, options); + for node in t.nodes().iter().filter(|n| n.token.is_some()) { + let derived = token_for(&prf(), &context, options, &node.positional).unwrap(); + assert_eq!(Some(derived), node.token, "{}", node.positional); + } + } + assert!(matches!( + token_for(&prf(), &context, JsonOptions::default(), &JsonPath::root()), + Err(TermError::JsonSealedRoot) + )); + assert!(token_for( + &prf(), + &context, + JsonOptions { sealed_root: false }, + &JsonPath::root() + ) + .is_ok()); + } + + #[test] + fn a_root_scalar_carries_its_element_selectors_as_alt() { + let scalar = terms(&json!("a"), JsonOptions::default()); + let array = terms(&json!(["a"]), JsonOptions::default()); + assert_eq!( + scalar.alt(), + Some((array.nodes()[1].s, array.nodes()[1].vs)), + "the alt pair is the stored array element's" + ); + assert!(terms(&json!({}), JsonOptions::default()).alt().is_none()); + } + + #[test] + fn container_elements_group_what_sits_inside_them() { + let t = terms(&json!({"a": [{"x": 1}, 2, [3]]}), JsonOptions::default()); + let nodes = t.nodes(); + // $, $.a, $.a[0], $.a[0].x, $.a[1], $.a[2], $.a[2][0] + assert!(nodes[2].container_element); + assert_eq!(nodes[3].element, Some(2)); + assert!( + !nodes[4].container_element, + "a scalar element groups nothing" + ); + assert_eq!(nodes[4].element, None); + assert!(nodes[5].container_element); + assert_eq!(nodes[6].element, Some(5)); + } + + #[test] + fn object_ordinals_are_a_permutation_that_does_not_rank_keys() { + let map: Map = ["a", "b", "c", "d"] + .iter() + .map(|k| (k.to_string(), json!(1))) + .collect(); + let context = Context::from_encoded(b"users/profile"); + let ordinals = shuffled_ordinals(&prf(), &context, &[1; 16], &map).unwrap(); + let mut seen: Vec = ordinals.values().copied().collect(); + seen.sort_unstable(); + assert_eq!(seen, [0, 1, 2, 3]); + let other = shuffled_ordinals(&prf(), &context, &[2; 16], &map).unwrap(); + assert_ne!( + ordinals, other, + "a different token shuffles differently (with overwhelming probability)" + ); + } + + /// A token's holder cannot test key names: the step from a parent's + /// token to a child's, and an object member's rank, are keyed with the + /// index key, so the same parent token and label give unrelated bytes + /// under another key, and nothing public reproduces them. + #[test] + fn the_token_step_and_the_rank_are_keyed() { + let context = Context::from_encoded(b"users/profile"); + let other = HmacSha256Prf::new(Protected::new([8; 32])); + let parent = [5; 16]; + let key = Label::Key("email@example.com".into()); + let step = |prf: &HmacSha256Prf, label: &Label| { + child_token(prf, &context, &parent, label).unwrap() + }; + assert_eq!(step(&prf(), &key), step(&prf(), &key), "deterministic"); + assert_ne!(step(&prf(), &key), step(&other, &key), "keyed"); + assert_ne!( + step(&prf(), &key), + h16(&[b"t", &parent, &[0x01], b"email@example.com"]), + "not the unkeyed hash" + ); + assert_ne!( + step(&prf(), &Label::Index(0)), + step(&prf(), &Label::Key("0".into())), + "an element is not a member" + ); + assert_ne!( + rank(&prf(), &context, &parent, "a").unwrap(), + rank(&other, &context, &parent, "a").unwrap(), + "the rank is keyed" + ); + } + + #[test] + fn a_node_encodes_and_decodes() { + for (label, scalar, kind) in [ + (Label::Root, None, Kind::Object), + (Label::Key("k".into()), Some(json!("v")), Kind::String), + (Label::Index(3), Some(json!(1.5)), Kind::Number), + (Label::Index(0), Some(json!(true)), Kind::Bool), + (Label::Key("n".into()), Some(Value::Null), Kind::Null), + (Label::Key("a".into()), None, Kind::Array), + ] { + let plain = NodePlain { + label: label.clone(), + kind, + scalar: scalar.clone(), + children: 2, + }; + let opened = decode_node(&encode_node(&plain)).unwrap(); + assert_eq!(opened.label, label); + assert_eq!(opened.kind, kind); + assert_eq!(opened.scalar, scalar); + assert_eq!(opened.children, 2); + } + assert!(decode_node(&[]).is_err()); + assert!(decode_node(&[9, 0, 0, 0, 0, 0]).is_err()); + + // The empty key is a key: its label is the tag and a zero length, + // and nothing after it. + let empty_key = NodePlain { + label: Label::Key(String::new()), + kind: Kind::Object, + scalar: None, + children: 0, + }; + let encoded = encode_node(&empty_key); + assert_eq!(encoded.len(), 1 + 4 + 1 + 4); + assert_eq!( + decode_node(&encoded).unwrap().label, + Label::Key(String::new()) + ); + // Every truncation of a labelled node is refused, not read past. + let keyed = encode_node(&NodePlain { + label: Label::Key("key".into()), + kind: Kind::Array, + scalar: None, + children: 1, + }); + let indexed = encode_node(&NodePlain { + label: Label::Index(7), + kind: Kind::Object, + scalar: None, + children: 1, + }); + for bytes in [&keyed, &indexed, &encoded] { + for len in 0..bytes.len() { + assert!(decode_node(&bytes[..len]).is_err(), "{len} of {bytes:?}"); + } + } + } + + #[test] + fn a_key_tag_must_fit_the_header() { + let value = json!({"a": 1}); + let context = Context::from_encoded(b"users/profile"); + let sealed = |tag_len: usize| { + let walked = walk(&value, JsonOptions::default(), &prf(), &context).unwrap(); + let key = stack_kms::DataKeyWithTag { + key: stack_kms::DataKey { + iv: [1; 16], + key: [2; 32], + }, + tag: vec![3; tag_len], + decryption_policy: None, + }; + seal( + walked, + Some(serde_json::to_vec(&value).unwrap()), + Uuid::from_u128(1), + key, + &context, + JsonOptions::default(), + ) + }; + let longest = usize::from(u16::MAX); + let doc = sealed(longest).unwrap(); + assert_eq!(doc.header().tag().len(), longest); + assert_eq!( + &JsonHeader::from_bytes(&doc.header().to_bytes()).unwrap(), + doc.header(), + "the longest tag the field holds round-trips" + ); + assert!(matches!(sealed(longest + 1), Err(Error::Other(_)))); + } + + #[test] + fn too_deep_a_document_is_refused() { + // The innermost scalar of `n` nested arrays sits at depth `n`. + let nested = |n: usize| { + let mut value = json!(1); + for _ in 0..n { + value = json!([value]); + } + value + }; + assert!(walk( + &nested(MAX_DEPTH), + JsonOptions::default(), + &prf(), + &Context::empty() + ) + .is_ok()); + assert!(matches!( + walk( + &nested(MAX_DEPTH + 1), + JsonOptions::default(), + &prf(), + &Context::empty() + ), + Err(TermError::JsonTooDeep) + )); + } +} diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs new file mode 100644 index 000000000..96085de76 --- /dev/null +++ b/packages/stack-encrypt/src/json/mod.rs @@ -0,0 +1,882 @@ +//! The JSON index: a document stored as one searchable value, with +//! node-local ciphertexts, a token chain and exact containment. +//! +//! `docs/plans/2026-10-17-json-tree-index.md` is the design +//! this module implements, and the authority on why. In short, a document +//! is flattened into one entry per node. Each entry carries: +//! +//! - a **selector** `s`, a PRF of the node's path with every array position +//! folded into the wildcard, so every element of an array shares one; +//! - a **value selector** `vs`, a PRF of the node's kind and, for a scalar, +//! its canonical value, bound to `s`; containment is a set comparison +//! over these two; +//! - the node's own **ciphertext** `c`, sealed under one ZeroKMS data key +//! per document: the node's label, its kind and, for a scalar, its +//! value, and for a container its child count. Nothing beneath a node is +//! encrypted in that node's entry, so a document's ciphertext is linear +//! in its size; +//! - the **token chain** fields `p` and `w`, which let the server walk the +//! subtree under a node from one client-derived token and nothing else, +//! and the **group tags** `n` and `g`, which tell the server which +//! entries sit inside the same array element, so containment agrees with +//! PostgreSQL's own `jsonb @>` across elements; +//! - a leaf ordering term `op` on number and string leaves, the same OPE +//! term the scalar [`Ope`](crate::target::Ope) index derives, keyed by +//! the column. +//! +//! By default the root is **sealed** ([`JsonOptions::sealed_root`]): the root +//! entry's ciphertext is the whole document, the top-level fields are the +//! heads of the chain, and no token exists for the root, so no query can +//! hand the server a token that unlocks the shape of every row in the +//! column. +//! +//! # What the types are +//! +//! [`Json`](struct@crate::target::Json) is the index, an [`Index`](crate::target::Index) +//! whose term is a [`JsonTerms`]: the one-way terms of a document (its +//! selectors, tokens and grouping), which a query is built from. Sealing is +//! a separate operation, [`json_document`](crate::target::json_document), +//! whose output is the stored [`JsonDocument`]; `JsonDocument` is itself an +//! [`EncryptFrom`](crate::EncryptFrom) target, so a plan or a record +//! names it by type. The query terms are [`JsonQuery`] (containment and +//! `AtPath`), [`JsonLeafQuery`] (ordering at a path), and the two +//! extraction keys a caller derives through the keyset cipher, +//! [`JsonSelector`] and [`JsonSubtreeKey`]. What the server returns for an +//! extraction opens through [`JsonNode`] (one entry) and [`JsonSubtree`] +//! (the entries under a token). +//! +//! # Where this deviates from the design +//! +//! Two deliberate deviations, recorded here so the plan can be read against +//! the code: +//! +//! - **Nonces are random, not derived.** The design derives each entry's +//! AES-GCM nonce from its selector and ordinal. This module seals every +//! entry through the engine's one AEAD path (`vitaminc-encrypt`'s +//! `Aes256Cipher`, a fresh random 96-bit nonce per seal, carried in the +//! ciphertext), so there is one leaf cipher in the crate and no second +//! AES-GCM call site. Uniqueness under one document key is probabilistic +//! rather than structural: at `N` entries the collision probability is +//! `N² / 2^97`, negligible for any document. The ordinal still binds the +//! entry to its place through the AAD, so a moved or reordered entry fails +//! to open as the design requires. The cost is twelve bytes per entry. +//! - **The ordinal field is `o`, not `i`**, because `i` is the EQL +//! envelope's identifier, which the SQL grafts onto an extracted entry. +//! +//! # Byte encodings +//! +//! The document's wire form is JSON, as the EQL domain stores it: +//! sixteen-byte fields (`s`, `vs`, `p`, `w`, `n`, `g`) and the OPE term `op` +//! in lowercase hex, the ciphertext `c` and the header `h` in standard +//! base64. The header's bytes are a frozen layout, [`JsonHeader`]. The +//! derivations are versioned by their PAE domain labels, as every other +//! term kind's are ([`sem`](crate::sem)): a change to any of them is a new +//! version, never an edit in place. +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use vitaminc_protected::OpaqueDebug; + +use crate::target::{DecryptField, DecryptInto, Decryptable, Decryption}; + +mod derive; +mod path; +mod query; + +pub use derive::JsonTerms; +pub use path::{JsonPath, JsonPathError, Segment}; +pub use query::{ + AtPathTerms, JsonLeafQuery, JsonProjection, JsonQuery, JsonSelector, JsonSubtreeKey, +}; + +pub(crate) use derive::{seal_pending, terms_pending}; + +/// A sixteen-byte tag: a selector, a token, a parent tag or a group tag. +pub type Tag = [u8; 16]; + +/// How a JSON index lays a document out. Part of the index's identity: a +/// stored document and the query probing it must agree on it, so it rides +/// in the data plan beside the index and is recorded in every document's +/// header. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct JsonOptions { + /// Seal the root: the root entry's ciphertext is the whole document and + /// the top-level fields are the heads of the token chain, so no token + /// for the root exists and no query can hand the server one. On by + /// default. Off, the chain starts at the root and a query may walk the + /// whole document from the root's token. + pub sealed_root: bool, +} + +impl Default for JsonOptions { + fn default() -> Self { + Self { sealed_root: true } + } +} + +/// The document's key header, stored once in `h`: what retrieving the +/// document's data key needs, and which shape the root has. +/// +/// # Frozen byte encoding +/// +/// | offset | field | size | value | +/// |---|---|---|---| +/// | 0 | header version | 1 | [`FORMAT_VERSION`](Self::FORMAT_VERSION) (`0x01`) | +/// | 1 | flags | 1 | bit 0: the root is sealed | +/// | 2 | epoch | 4 | `u32` little-endian; reserved, always `0` | +/// | 6 | keyset id | 16 | the keyset the data key was minted under, raw UUID bytes | +/// | 22 | ZeroKMS `iv` | 16 | identifies the data key for retrieval; also the per-document salt of the chain tags | +/// | 38 | `tag_len` | 2 | `u16` little-endian | +/// | 40 | ZeroKMS key `tag` | `tag_len` | required to retrieve the key | +/// +/// The SQL reads the `iv` at offset 22 to compute parent and group tags; it +/// parses nothing else. The epoch is reserved for bounding a revealed token +/// in time (the design's "Epochs"); writing it now keeps that a new value +/// rather than a new layout. +/// +/// `Debug` ([`OpaqueDebug`]) prints the keyset and the layout, which say +/// what a header is for, and masks the ZeroKMS `iv` and `tag`, which +/// retrieve the data key. +#[derive(Clone, PartialEq, Eq, OpaqueDebug)] +pub struct JsonHeader { + #[non_sensitive] + keyset_id: Uuid, + iv: stack_kms::Iv, + tag: Vec, + #[non_sensitive] + sealed_root: bool, + #[non_sensitive] + epoch: u32, +} + +/// The [`JsonHeader`] bytes are not the frozen layout. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error, miette::Diagnostic)] +#[non_exhaustive] +pub enum JsonHeaderError { + /// The leading version byte is not one this build parses. + #[error("unknown JSON header version {0}")] + #[diagnostic(code(stack_encrypt::json_header_version))] + UnknownVersion(u8), + /// The bytes end before the fixed fields, or before the tag promised. + #[error("JSON header bytes are truncated")] + #[diagnostic(code(stack_encrypt::json_header_truncated))] + Truncated, +} + +impl JsonHeader { + /// The version byte of the frozen layout. + pub const FORMAT_VERSION: u8 = 1; + const FLAG_SEALED_ROOT: u8 = 0b1; + const FIXED_LEN: usize = 1 + 1 + 4 + 16 + 16 + 2; + + /// The keyset the document's data key was minted under. + pub fn keyset_id(&self) -> Uuid { + self.keyset_id + } + + /// The ZeroKMS IV of the document's data key: its identifier, and the + /// per-document salt of every chain tag. + pub fn iv(&self) -> &stack_kms::Iv { + &self.iv + } + + /// The ZeroKMS key tag. + pub fn tag(&self) -> &[u8] { + &self.tag + } + + /// Whether the root is sealed: see [`JsonOptions::sealed_root`]. + pub fn sealed_root(&self) -> bool { + self.sealed_root + } + + /// The layout this document was laid out under. + pub fn options(&self) -> JsonOptions { + JsonOptions { + sealed_root: self.sealed_root, + } + } + + /// The frozen byte layout; see the type docs. + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::with_capacity(Self::FIXED_LEN + self.tag.len()); + out.push(Self::FORMAT_VERSION); + out.push(if self.sealed_root { + Self::FLAG_SEALED_ROOT + } else { + 0 + }); + out.extend_from_slice(&self.epoch.to_le_bytes()); + out.extend_from_slice(self.keyset_id.as_bytes()); + out.extend_from_slice(&self.iv); + // `seal_pending` refuses a tag longer than the field, so the + // saturation is unreachable; it is a bound, not a case. + out.extend_from_slice( + &u16::try_from(self.tag.len()) + .unwrap_or(u16::MAX) + .to_le_bytes(), + ); + out.extend_from_slice(&self.tag); + out + } + + /// Parse the frozen byte layout. Structural only: nothing here is + /// trusted until an entry opens under the key it names. + pub fn from_bytes(bytes: &[u8]) -> Result { + let (&version, rest) = bytes.split_first().ok_or(JsonHeaderError::Truncated)?; + if version != Self::FORMAT_VERSION { + return Err(JsonHeaderError::UnknownVersion(version)); + } + if rest.len() < Self::FIXED_LEN - 1 { + return Err(JsonHeaderError::Truncated); + } + let (flags, rest) = rest.split_at(1); + let (epoch, rest) = rest.split_at(4); + let (keyset, rest) = rest.split_at(16); + let (iv, rest) = rest.split_at(16); + let (tag_len, rest) = rest.split_at(2); + let tag_len = usize::from(u16::from_le_bytes([tag_len[0], tag_len[1]])); + if rest.len() < tag_len { + return Err(JsonHeaderError::Truncated); + } + let mut iv_bytes: stack_kms::Iv = [0; 16]; + iv_bytes.copy_from_slice(iv); + Ok(Self { + keyset_id: Uuid::from_slice(keyset).map_err(|_| JsonHeaderError::Truncated)?, + iv: iv_bytes, + tag: rest[..tag_len].to_vec(), + sealed_root: flags[0] & Self::FLAG_SEALED_ROOT != 0, + epoch: u32::from_le_bytes([epoch[0], epoch[1], epoch[2], epoch[3]]), + }) + } + + pub(crate) fn new(keyset_id: Uuid, iv: stack_kms::Iv, tag: Vec, sealed_root: bool) -> Self { + Self { + keyset_id, + iv, + tag, + sealed_root, + epoch: 0, + } + } +} + +/// One entry of a stored document: one JSON node. +/// +/// Every field but `s`, `vs` and `c` is optional, and which are present +/// says what the node is: `p`, `w` and `o` on a node whose parent has a +/// token (everything below the chain's heads); `n` on an array element +/// that is a container; `g` on every entry inside such an element; `op` on +/// a number or string leaf. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JsonEntry { + s: Tag, + vs: Tag, + c: Vec, + o: Option, + p: Option, + w: Option, + n: Option, + g: Option, + op: Option>, +} + +impl JsonEntry { + /// The path selector. + pub fn selector(&self) -> &Tag { + &self.s + } + /// The value selector. + pub fn value_selector(&self) -> &Tag { + &self.vs + } + /// The node's ciphertext, as sealed. + pub fn ciphertext(&self) -> &[u8] { + &self.c + } + /// The node's ordinal under its parent, when its parent has a token. + pub fn ordinal(&self) -> Option { + self.o + } + /// The parent tag, when the parent has a token. + pub fn parent_tag(&self) -> Option<&Tag> { + self.p.as_ref() + } + /// The wrapped token, when the parent has a token. + pub fn wrapped_token(&self) -> Option<&Tag> { + self.w.as_ref() + } + /// This node's group tag, when it is an array element that is a + /// container. + pub fn group_tag(&self) -> Option<&Tag> { + self.n.as_ref() + } + /// The enclosing array element's group tag, when this node sits inside + /// an element that is a container. + pub fn enclosing_group_tag(&self) -> Option<&Tag> { + self.g.as_ref() + } + /// The leaf ordering term, on a number or string leaf. + pub fn ordering_term(&self) -> Option<&[u8]> { + self.op.as_deref() + } +} + +/// A stored JSON document: its [`JsonHeader`] and its entries, one per +/// node, in depth-first pre-order with the root first. What +/// [`json_document`](crate::target::json_document) produces and +/// [`DecryptInto`](crate::DecryptInto) opens. +/// +/// Serializes to the wire form the EQL domain stores: `{"h": "…", "sv": +/// [{"s": "…", "vs": "…", "c": "…", …}, …]}`; see the +/// [module docs](self#byte-encodings). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "DocumentWire", into = "DocumentWire")] +pub struct JsonDocument { + header: JsonHeader, + entries: Vec, +} + +impl JsonDocument { + /// The document's header. + pub fn header(&self) -> &JsonHeader { + &self.header + } + + /// The entries, in depth-first pre-order; the root is first. + pub fn entries(&self) -> &[JsonEntry] { + &self.entries + } + + /// The children of the node whose token is `token`, as the server + /// finds them: every entry whose parent tag is the tag of `token` at + /// the entry's own ordinal, with its own token unwrapped, in ordinal + /// order. The SQL walk finds the same entries by trying the ordinals + /// `0..F`, where `F` is one more than the largest ordinal in the + /// document; checking each entry at its own ordinal gives the same + /// answer without a loop whose length a stored document chooses. A + /// token that names no node yields nothing. + /// + /// What a `jsonb_array_elements(doc, token)` returns; one level of the + /// walk [`subtree`](Self::subtree) runs to the bottom. + pub fn children(&self, token: &Tag) -> Vec<(usize, &JsonEntry, Tag)> { + let iv = &self.header.iv; + let mut out: Vec<(usize, &JsonEntry, Tag)> = self + .entries + .iter() + .enumerate() + .filter_map(|(j, e)| { + let (p, w, o) = (e.p.as_ref()?, e.w.as_ref()?, e.o?); + (derive::parent_tag(iv, token, o) == *p).then(|| { + let child = derive::xor(w, &derive::mask(iv, token, &e.c)); + (j, e, child) + }) + }) + .collect(); + out.sort_by_key(|(_, e, _)| e.o); + out + } + + /// Every entry under the node whose token is `token`, with its depth + /// below that node, as the server's recursive walk finds them: the node + /// itself is not included, since the walk addresses nodes by their + /// parent. In depth-first order, siblings by ordinal. + /// + /// A stored document is untrusted, and its tags need not form a tree: + /// whoever holds a token can write a chain deeper than any document the + /// index seals, or a cycle. The walk stops descending below the deepest + /// level a sealed document has, and stops once it has returned as many + /// entries as the document holds, which a tree never exceeds, so it + /// ends, and in bounded stack, on any document. + pub fn subtree(&self, token: &Tag) -> Vec<(usize, &JsonEntry, u32)> { + let mut out = Vec::new(); + self.walk(token, 1, &mut out); + out + } + + fn walk<'a>(&'a self, token: &Tag, depth: u32, out: &mut Vec<(usize, &'a JsonEntry, u32)>) { + if depth as usize > derive::MAX_DEPTH { + return; + } + for (j, entry, child) in self.children(token) { + if out.len() >= self.entries.len() { + return; + } + out.push((j, entry, depth)); + self.walk(&child, depth + 1, out); + } + } + + /// The node at ordinal `j` as an extraction returns it: the entry with + /// the header beside it, which opens on its own. + pub fn node(&self, j: usize) -> Option { + self.entries.get(j).map(|entry| JsonNode { + entry: entry.clone(), + header: self.header.clone(), + j: j as u64, + depth: None, + }) + } + + pub(crate) fn new(header: JsonHeader, entries: Vec) -> Self { + Self { header, entries } + } +} + +impl Decryptable for JsonDocument { + const DECRYPTABLE: bool = true; +} +impl DecryptField for JsonDocument +where + Self: crate::DecryptInto

, + Ctx: Into<>::Context>, +{ + fn decryption_field(self, context: Ctx) -> Option> { + Some(self.decryption(context.into())) + } +} + +/// One entry as an extraction returns it: the entry, the document's header +/// grafted beside it, and the entry's ordinal `j` in the document, which +/// its ciphertext is bound to. What `->` and the walk return per row, and +/// what [`DecryptInto`](crate::DecryptInto) opens to the node's +/// scalar, or to the whole document for a sealed root. +/// +/// Deserializes from the entry's wire fields plus `h` and `j`, and tolerates +/// the envelope fields the SQL grafts beside them (`i`, `v`, `d`). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "NodeWire", into = "NodeWire")] +pub struct JsonNode { + entry: JsonEntry, + header: JsonHeader, + j: u64, + depth: Option, +} + +impl JsonNode { + /// The entry. + pub fn entry(&self) -> &JsonEntry { + &self.entry + } + /// The document's header. + pub fn header(&self) -> &JsonHeader { + &self.header + } + /// The entry's ordinal in the document. + pub fn ordinal(&self) -> u64 { + self.j + } + /// The entry's depth below the walk's start, when the walk reported one. + pub fn depth(&self) -> Option { + self.depth + } +} + +impl Decryptable for JsonNode { + const DECRYPTABLE: bool = true; +} +impl DecryptField for JsonNode +where + Self: crate::DecryptInto

, + Ctx: Into<>::Context>, +{ + fn decryption_field(self, context: Ctx) -> Option> { + Some(self.decryption(context.into())) + } +} + +/// The entries the server's walk returned for one token: what +/// `json_tree_subtree(doc, token)` yields, collected, and what +/// [`DecryptInto`](crate::DecryptInto) reassembles into the value +/// under that node. +/// +/// The walk returns the node's descendants and not the node itself, so the +/// node's kind is read from its children's labels: elements make an array, +/// members an object. A node with no descendants, an empty container or a +/// scalar, yields no rows and so no value; `->` on its selector returns the +/// node itself. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct JsonSubtree { + nodes: Vec, +} + +impl JsonSubtree { + /// The rows, as returned. + pub fn new(nodes: Vec) -> Self { + Self { nodes } + } + + /// The rows. + pub fn nodes(&self) -> &[JsonNode] { + &self.nodes + } +} + +impl Decryptable for JsonSubtree { + const DECRYPTABLE: bool = true; +} +impl DecryptField for JsonSubtree +where + Self: crate::DecryptInto

, + Ctx: Into<>::Context>, +{ + fn decryption_field(self, context: Ctx) -> Option> { + Some(self.decryption(context.into())) + } +} + +// ============================================================================= +// Wire form +// ============================================================================= + +/// Lowercase hex, as the SQL decodes with `decode(x, 'hex')`. +pub(crate) fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &b in bytes { + out.push(DIGITS[usize::from(b >> 4)] as char); + out.push(DIGITS[usize::from(b & 0xf)] as char); + } + out +} + +/// The inverse of [`hex`]; either case. +pub(crate) fn unhex(text: &str) -> Result, WireError> { + fn nibble(c: u8) -> Result { + match c { + b'0'..=b'9' => Ok(c - b'0'), + b'a'..=b'f' => Ok(c - b'a' + 10), + b'A'..=b'F' => Ok(c - b'A' + 10), + _ => Err(WireError("not hex")), + } + } + let bytes = text.as_bytes(); + if !bytes.len().is_multiple_of(2) { + return Err(WireError("odd hex length")); + } + bytes + .chunks_exact(2) + .map(|pair| Ok(nibble(pair[0])? << 4 | nibble(pair[1])?)) + .collect() +} + +fn unhex16(text: &str) -> Result { + Tag::try_from(unhex(text)?).map_err(|_| WireError("not 16 bytes")) +} + +fn base64(bytes: &[u8]) -> String { + use base64ct::{Base64, Encoding}; + Base64::encode_string(bytes) +} + +fn unbase64(text: &str) -> Result, WireError> { + use base64ct::{Base64, Encoding}; + Base64::decode_vec(text).map_err(|_| WireError("not base64")) +} + +/// A wire field that does not decode. Names the field's fault and never its +/// text. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("a JSON index wire field does not decode: {0}")] +pub struct WireError(&'static str); + +/// `h` and `sv`, as stored. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DocumentWire { + h: String, + sv: Vec, +} + +/// One entry, as stored: tags in hex, the ciphertext in base64. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EntryWire { + s: String, + vs: String, + c: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + o: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + w: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + n: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + g: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + op: Option, +} + +impl From<&JsonEntry> for EntryWire { + fn from(e: &JsonEntry) -> Self { + Self { + s: hex(&e.s), + vs: hex(&e.vs), + c: base64(&e.c), + o: e.o, + p: e.p.as_ref().map(|t| hex(t)), + w: e.w.as_ref().map(|t| hex(t)), + n: e.n.as_ref().map(|t| hex(t)), + g: e.g.as_ref().map(|t| hex(t)), + op: e.op.as_deref().map(hex), + } + } +} + +impl TryFrom for JsonEntry { + type Error = WireError; + + fn try_from(w: EntryWire) -> Result { + Ok(Self { + s: unhex16(&w.s)?, + vs: unhex16(&w.vs)?, + c: unbase64(&w.c)?, + o: w.o, + p: w.p.as_deref().map(unhex16).transpose()?, + w: w.w.as_deref().map(unhex16).transpose()?, + n: w.n.as_deref().map(unhex16).transpose()?, + g: w.g.as_deref().map(unhex16).transpose()?, + op: w.op.as_deref().map(unhex).transpose()?, + }) + } +} + +impl From for DocumentWire { + fn from(doc: JsonDocument) -> Self { + Self { + h: base64(&doc.header.to_bytes()), + sv: doc.entries.iter().map(EntryWire::from).collect(), + } + } +} + +impl TryFrom for JsonDocument { + type Error = WireError; + + fn try_from(w: DocumentWire) -> Result { + let header = + JsonHeader::from_bytes(&unbase64(&w.h)?).map_err(|_| WireError("not a header"))?; + let entries = + w.sv.into_iter() + .map(JsonEntry::try_from) + .collect::>()?; + Ok(Self { header, entries }) + } +} + +/// An extracted entry: the entry's fields, the grafted header and ordinal, +/// and whatever else the SQL grafted (`i`, `v`), which is ignored. +#[derive(Clone, Serialize, Deserialize)] +struct NodeWire { + #[serde(flatten)] + entry: EntryWire, + h: String, + j: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + d: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + i: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + v: Option, +} + +impl From for NodeWire { + fn from(node: JsonNode) -> Self { + Self { + entry: EntryWire::from(&node.entry), + h: base64(&node.header.to_bytes()), + j: node.j, + d: node.depth, + i: None, + v: None, + } + } +} + +impl TryFrom for JsonNode { + type Error = WireError; + + fn try_from(w: NodeWire) -> Result { + Ok(Self { + entry: JsonEntry::try_from(w.entry)?, + header: JsonHeader::from_bytes(&unbase64(&w.h)?) + .map_err(|_| WireError("not a header"))?, + j: w.j, + depth: w.d, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_header_round_trips_its_frozen_layout() { + let header = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![1, 2, 3], true); + let bytes = header.to_bytes(); + assert_eq!(bytes[0], JsonHeader::FORMAT_VERSION); + assert_eq!(bytes[1], 1, "the sealed-root flag"); + assert_eq!(&bytes[2..6], &[0; 4], "the epoch is reserved at zero"); + assert_eq!( + &bytes[22..38], + &[9; 16], + "the iv sits at offset 22, where the SQL reads it" + ); + assert_eq!(JsonHeader::from_bytes(&bytes).unwrap(), header); + let open = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![], false); + assert_eq!(JsonHeader::from_bytes(&open.to_bytes()).unwrap(), open); + assert!(!JsonHeader::from_bytes(&open.to_bytes()) + .unwrap() + .sealed_root()); + assert!(matches!( + JsonHeader::from_bytes(&bytes[..10]), + Err(JsonHeaderError::Truncated) + )); + assert!(matches!( + JsonHeader::from_bytes(&[2]), + Err(JsonHeaderError::UnknownVersion(2)) + )); + } + + #[test] + fn a_header_reads_back_what_it_was_made_with() { + let header = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![1, 2, 3], false); + assert_eq!(header.keyset_id(), Uuid::from_u128(7)); + assert_eq!(header.iv(), &[9; 16]); + assert_eq!(header.tag(), &[1, 2, 3]); + assert!(!header.sealed_root()); + assert_eq!(header.options(), JsonOptions { sealed_root: false }); + let sealed = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![], true); + assert_eq!(sealed.options(), JsonOptions { sealed_root: true }); + } + + #[test] + fn every_truncation_of_a_header_is_refused() { + let header = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![1, 2, 3], true); + let bytes = header.to_bytes(); + assert_eq!(bytes.len(), 40 + 3, "the fixed fields, then the tag"); + // Short of the fixed fields, and short of the tag `tag_len` promises. + for len in 0..bytes.len() { + assert_eq!( + JsonHeader::from_bytes(&bytes[..len]), + Err(JsonHeaderError::Truncated), + "{len} bytes" + ); + } + let bare = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![], true); + assert_eq!( + JsonHeader::from_bytes(&bare.to_bytes()), + Ok(bare), + "the fixed fields alone are a header with an empty tag" + ); + } + + #[test] + fn a_header_prints_its_keyset_and_layout_and_masks_the_key_identifiers() { + let header = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![1, 2, 3], true); + assert_eq!( + format!("{header:?}"), + format!( + "stack_encrypt::json::JsonHeader {{ keyset_id: {:?}, iv: \"***\", tag: \"***\", sealed_root: true, epoch: 0 }}", + Uuid::from_u128(7) + ) + ); + } + + /// A chained entry that unwraps to `child` under `parent` at ordinal `o`, + /// as whoever holds `parent` can write one. + fn chained(iv: &[u8], parent: &Tag, o: u32, child: &Tag, c: Vec) -> JsonEntry { + JsonEntry { + s: [0; 16], + vs: [0; 16], + o: Some(o), + p: Some(derive::parent_tag(iv, parent, o)), + w: Some(derive::xor(child, &derive::mask(iv, parent, &c))), + n: None, + g: None, + op: None, + c, + } + } + + fn root() -> JsonEntry { + JsonEntry { + s: [0; 16], + vs: [0; 16], + c: vec![0], + o: None, + p: None, + w: None, + n: None, + g: None, + op: None, + } + } + + fn document(entries: Vec) -> JsonDocument { + JsonDocument::new( + JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![], true), + entries, + ) + } + + #[test] + fn an_ordinal_at_the_top_of_its_range_is_an_entry_like_any_other() { + let token = [1; 16]; + let mut entry = chained(&[9; 16], &token, u32::MAX, &[2; 16], vec![3; 20]); + let doc = document(vec![root(), entry.clone()]); + let children = doc.children(&token); + assert_eq!(children.len(), 1, "found at its own ordinal, no fan-out"); + assert_eq!(children[0].2, [2; 16]); + assert!(doc.children(&[7; 16]).is_empty()); + entry.p = Some([0; 16]); + assert!(document(vec![root(), entry]).children(&token).is_empty()); + } + + #[test] + fn a_walk_ends_on_tags_that_do_not_form_a_tree() { + let iv = [9; 16]; + let token = [1; 16]; + // Two children of the token that each unwrap to the token itself: + // a cycle that doubles at every level. + let cycle = document(vec![ + root(), + chained(&iv, &token, 0, &token, vec![4; 20]), + chained(&iv, &token, 1, &token, vec![5; 20]), + ]); + assert_eq!( + cycle.subtree(&token).len(), + cycle.entries().len(), + "no more entries than the document holds" + ); + + // A chain longer than any document the index seals. + let mut entries = vec![root()]; + let mut parent = token; + for i in 0..1000u32 { + let mut child = [0; 16]; + child[..4].copy_from_slice(&i.to_le_bytes()); + child[15] = 0xee; + entries.push(chained(&iv, &parent, 0, &child, i.to_le_bytes().to_vec())); + parent = child; + } + let chain = document(entries); + let walked = chain.subtree(&token); + assert_eq!(walked.len(), derive::MAX_DEPTH, "cut at the deepest level"); + assert_eq!( + walked.last().map(|(_, _, depth)| *depth as usize), + Some(derive::MAX_DEPTH) + ); + } + + #[test] + fn hex_round_trips_and_refuses_what_is_not_hex() { + assert_eq!(hex(&[0, 15, 255]), "000fff"); + assert_eq!(unhex("000FfF").unwrap(), vec![0, 15, 255]); + assert!(unhex("abc").is_err()); + assert!(unhex("zz").is_err()); + } +} diff --git a/packages/stack-encrypt/src/json/path.rs b/packages/stack-encrypt/src/json/path.rs new file mode 100644 index 000000000..968747b71 --- /dev/null +++ b/packages/stack-encrypt/src/json/path.rs @@ -0,0 +1,424 @@ +//! JSON paths: how a caller names a node of a document. +//! +//! A [`JsonPath`] is parsed from the JSONPath subset the JSON index +//! addresses: `$`, object members (`.name` or `["name"]`), array elements +//! by position (`[2]`) and every element of an array (`[*]`). Nothing else: +//! no filters, no recursive descent, no slices. +//! +//! The index derives two things from a path, and they differ on arrays: +//! +//! - A **selector** names a path with every array position replaced by the +//! wildcard, because every element of an array shares one selector +//! ([`JsonPath::selector_bytes`]). `$.a[2].x` and `$.a[*].x` have the same +//! selector. +//! - A **token** names a path positionally ([`JsonPath::token_bytes`]), so a +//! path with a wildcard has none: the token chain addresses one node. +use std::fmt; +use std::str::FromStr; + +/// One step of a [`JsonPath`]. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Segment { + /// An object member, by key. + Key(String), + /// One array element, by position. + Index(u64), + /// Every element of an array. + Wildcard, +} + +/// A path into a JSON document, from the root (`$`) down. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] +pub struct JsonPath { + segments: Vec, +} + +/// Why text is not a [`JsonPath`]. The message names the position of the +/// first character the parser refused and nothing else of the text: a path +/// can name a key that is customer data. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error, miette::Diagnostic)] +#[error("not a JSON path: {reason} at byte {at}")] +#[diagnostic( + code(stack_encrypt::json_path), + help("A path starts with `$` and continues with `.name`, `[\"name\"]`, `[2]` or `[*]` steps, as in `$.address.city` or `$.tags[*]`.") +)] +pub struct JsonPathError { + /// What was expected, or what was found. + pub reason: &'static str, + /// The byte offset in the text the parser stopped at. + pub at: usize, +} + +impl crate::ErrorPayload for JsonPathError { + fn payload(&self) -> serde_json::Map { + crate::diagnostic::payload([ + ("reason", self.reason.into()), + ("at", (self.at as u64).into()), + ]) + } +} + +/// The tag byte of an object step in a path's canonical bytes. +const KEY_STEP: u8 = 0x01; +/// The tag byte of an array step (positional or wildcard) in a path's +/// canonical bytes. +const ARRAY_STEP: u8 = 0x02; + +impl JsonPath { + /// The root, `$`. + pub fn root() -> Self { + Self::default() + } + + /// Parse the JSONPath subset the index addresses: `$`, `.name`, + /// `["name"]`, `[2]` and `[*]` steps. + /// + /// # Errors + /// + /// [`JsonPathError`] for text that is not one. + pub fn parse(text: &str) -> Result { + // The parser walks a cursor, `rest`: the text not yet read. Every + // step strips at least one byte from its front, so the loop ends on + // any input, and a refusal names the offset of what was left. + let at = |rest: &str| text.len() - rest.len(); + let refuse = |reason, rest: &str| { + Err(JsonPathError { + reason, + at: at(rest), + }) + }; + let Some(mut rest) = text.strip_prefix('$') else { + return refuse("expected `$`", text); + }; + let mut segments = Vec::new(); + while !rest.is_empty() { + if let Some(after) = rest.strip_prefix('.') { + let end = after.find(['.', '[']).unwrap_or(after.len()); + if end == 0 { + return refuse("expected a member name after `.`", after); + } + // `end` is at an ASCII delimiter or the end of the text, so + // the split falls on a character boundary. + let (name, next) = after.split_at(end); + segments.push(Segment::Key(name.to_owned())); + rest = next; + } else if let Some(after) = rest.strip_prefix('[') { + if let Some(star) = after.strip_prefix('*') { + let Some(next) = star.strip_prefix(']') else { + return refuse("expected `]` after `*`", star); + }; + segments.push(Segment::Wildcard); + rest = next; + } else if after.starts_with('"') { + let (key, next) = match quoted(after) { + Ok(parts) => parts, + Err(reason) => return refuse(reason, after), + }; + let Some(next) = next.strip_prefix(']') else { + return refuse("expected `]` after the quoted name", next); + }; + segments.push(Segment::Key(key)); + rest = next; + } else if after.starts_with(|c: char| c.is_ascii_digit()) { + let end = after + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(after.len()); + let (digits, next) = after.split_at(end); + let Some(next) = next.strip_prefix(']') else { + return refuse("expected `]` after the position", next); + }; + let Ok(index) = digits.parse::() else { + return refuse("the position does not fit 64 bits", after); + }; + segments.push(Segment::Index(index)); + rest = next; + } else { + return refuse("expected `*`, a quoted name or a position after `[`", after); + } + } else { + return refuse("expected `.` or `[`", rest); + } + } + Ok(Self { segments }) + } + + /// The steps, from the root down. Empty for the root itself. + pub fn segments(&self) -> &[Segment] { + &self.segments + } + + /// Whether this path names one node: no step is the wildcard. + pub fn is_positional(&self) -> bool { + !self.segments.iter().any(|s| matches!(s, Segment::Wildcard)) + } + + /// This path with one more step. + pub(crate) fn then(&self, segment: Segment) -> Self { + let mut segments = Vec::with_capacity(self.segments.len() + 1); + segments.extend_from_slice(&self.segments); + segments.push(segment); + Self { segments } + } + + /// The longest prefix with no wildcard, and whether anything was cut. + pub(crate) fn anchor(&self) -> (Self, bool) { + let end = self + .segments + .iter() + .position(|s| matches!(s, Segment::Wildcard)) + .unwrap_or(self.segments.len()); + ( + Self { + segments: self.segments[..end].to_vec(), + }, + end < self.segments.len(), + ) + } + + /// The canonical bytes a selector is derived over: every array step, + /// positional or wildcard, is the one array tag, so every element of an + /// array shares its parent's selector. A key is length-prefixed, so two + /// paths cannot encode to one byte string. + pub(crate) fn selector_bytes(&self) -> Vec { + let mut out = Vec::new(); + for segment in &self.segments { + match segment { + Segment::Key(key) => push_key(&mut out, key), + Segment::Index(_) | Segment::Wildcard => out.push(ARRAY_STEP), + } + } + out + } + + /// The canonical bytes a token is derived over: an array step carries + /// its position. `None` for a path with a wildcard, which names no one + /// node. + pub(crate) fn token_bytes(&self) -> Option> { + let mut out = Vec::new(); + for segment in &self.segments { + match segment { + Segment::Key(key) => push_key(&mut out, key), + Segment::Index(index) => { + out.push(ARRAY_STEP); + out.extend_from_slice(&index.to_le_bytes()); + } + Segment::Wildcard => return None, + } + } + Some(out) + } + + /// The document that holds `value` at this path and nothing else: + /// `$.a[*].x` with `1` is `{"a": [{"x": 1}]}`. A positional array step + /// places the value at element zero, since the needle's selectors do not + /// tell positions apart. + pub(crate) fn needle(&self, value: serde_json::Value) -> serde_json::Value { + self.segments + .iter() + .rev() + .fold(value, |inner, segment| match segment { + Segment::Key(key) => { + let mut map = serde_json::Map::new(); + let _ = map.insert(key.clone(), inner); + serde_json::Value::Object(map) + } + Segment::Index(_) | Segment::Wildcard => serde_json::Value::Array(vec![inner]), + }) + } +} + +/// A key step: the tag, the key's length and the key. +fn push_key(out: &mut Vec, key: &str) { + out.push(KEY_STEP); + out.extend_from_slice(&(key.len() as u32).to_le_bytes()); + out.extend_from_slice(key.as_bytes()); +} + +/// The JSON string at the front of `text`, which starts at its opening +/// quote: its value and the text after its closing quote. The escapes are +/// JSON's. The refusal is the reason; the caller names the offset. +fn quoted(text: &str) -> Result<(String, &str), &'static str> { + let mut escaped = false; + for (end, byte) in text.bytes().enumerate().skip(1) { + match (escaped, byte) { + (true, _) => escaped = false, + (false, b'\\') => escaped = true, + (false, b'"') => { + let key: String = serde_json::from_str(&text[..=end]) + .map_err(|_| "the quoted name is not a JSON string")?; + return Ok((key, &text[end + 1..])); + } + (false, _) => {} + } + } + Err("unterminated quoted name") +} + +impl FromStr for JsonPath { + type Err = JsonPathError; + + fn from_str(text: &str) -> Result { + Self::parse(text) + } +} + +/// Renders the path as it parses: a plain key as `.key`, any other as +/// `["…"]`, a position as `[2]`, the wildcard as `[*]`. +impl fmt::Display for JsonPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("$")?; + for segment in &self.segments { + match segment { + Segment::Key(key) if !key.is_empty() && !key.contains(['.', '[', ']', '"']) => { + write!(f, ".{key}")?; + } + Segment::Key(key) => { + write!(f, "[{}]", serde_json::Value::String(key.clone()))?; + } + Segment::Index(index) => write!(f, "[{index}]")?, + Segment::Wildcard => f.write_str("[*]")?, + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(k: &str) -> Segment { + Segment::Key(k.to_owned()) + } + + #[test] + fn parses_the_subset_the_index_addresses() { + let path = JsonPath::parse("$.a[2].b[*][\"c.d\"]").unwrap(); + assert_eq!( + path.segments(), + &[ + key("a"), + Segment::Index(2), + key("b"), + Segment::Wildcard, + key("c.d") + ] + ); + assert_eq!(path.to_string(), "$.a[2].b[*][\"c.d\"]"); + assert_eq!(JsonPath::parse("$").unwrap(), JsonPath::root()); + assert_eq!(JsonPath::parse("$").unwrap().to_string(), "$"); + assert!(!path.is_positional()); + assert!(JsonPath::parse("$.a[2]").unwrap().is_positional()); + } + + #[test] + fn refuses_what_it_does_not_address() { + for (text, at) in [ + ("", 0), + ("a.b", 0), + ("$.", 2), + ("$..a", 2), + ("$[", 2), + ("$[1", 3), + ("$[-1]", 2), + ("$[\"a]", 2), + ("$.a[x]", 4), + ("$[*x", 3), + ("$[*", 3), + ("$[\"a\"x", 5), + ("$[\"a\"", 5), + ("$[\"\\q\"]", 2), + ("$[18446744073709551616]", 2), + ] { + let error = JsonPath::parse(text).unwrap_err(); + assert_eq!(error.at, at, "{text:?}: {error}"); + } + // A plain key runs to the next `.` or `[`, so it may hold a space. + assert_eq!(JsonPath::parse("$.a b").unwrap().segments(), &[key("a b")]); + } + + #[test] + fn a_quoted_name_takes_json_escapes() { + assert_eq!( + JsonPath::parse(r#"$["a\"b"]"#).unwrap().segments(), + &[key("a\"b")], + "an escaped quote does not end the name" + ); + assert_eq!( + JsonPath::parse(r#"$["a\\"].b"#).unwrap().segments(), + &[key("a\\"), key("b")], + "an escaped backslash does not escape the closing quote" + ); + assert_eq!( + JsonPath::parse(r#"$["\u0041"][0]"#).unwrap().segments(), + &[key("A"), Segment::Index(0)] + ); + assert_eq!( + JsonPath::parse("$[18446744073709551615]") + .unwrap() + .segments(), + &[Segment::Index(u64::MAX)] + ); + } + + #[test] + fn a_path_parses_through_from_str() { + let path: JsonPath = "$.a[*]".parse().unwrap(); + assert_eq!(path, JsonPath::parse("$.a[*]").unwrap()); + assert_eq!("$.".parse::().unwrap_err().at, 2); + } + + #[test] + fn a_refusal_reports_its_reason_and_offset_and_not_the_text() { + use crate::ErrorPayload; + let error = JsonPath::parse("$.secret[x]").unwrap_err(); + let payload = error.payload(); + assert_eq!( + payload.get("reason"), + Some(&serde_json::json!( + "expected `*`, a quoted name or a position after `[`" + )) + ); + assert_eq!(payload.get("at"), Some(&serde_json::json!(9))); + assert_eq!(payload.len(), 2, "{payload:?}"); + assert!(!error.to_string().contains("secret")); + } + + #[test] + fn a_selector_folds_positions_into_the_wildcard_and_a_token_keeps_them() { + let positional = JsonPath::parse("$.a[2].x").unwrap(); + let wild = JsonPath::parse("$.a[*].x").unwrap(); + assert_eq!(positional.selector_bytes(), wild.selector_bytes()); + assert!(positional.token_bytes().is_some()); + assert!(wild.token_bytes().is_none()); + assert_ne!( + JsonPath::parse("$.a[1]").unwrap().token_bytes(), + JsonPath::parse("$.a[2]").unwrap().token_bytes() + ); + // Length-prefixed keys: `("ab", "c")` and `("a", "bc")` differ. + assert_ne!( + JsonPath::parse("$.ab.c").unwrap().selector_bytes(), + JsonPath::parse("$.a.bc").unwrap().selector_bytes() + ); + } + + #[test] + fn the_anchor_is_the_prefix_before_the_first_wildcard() { + let (anchor, cut) = JsonPath::parse("$.a[*].x").unwrap().anchor(); + assert_eq!(anchor, JsonPath::parse("$.a").unwrap()); + assert!(cut); + let (anchor, cut) = JsonPath::parse("$.a[1].x").unwrap().anchor(); + assert_eq!(anchor.to_string(), "$.a[1].x"); + assert!(!cut); + } + + #[test] + fn a_needle_holds_the_value_at_the_path() { + let needle = JsonPath::parse("$.a[*].x") + .unwrap() + .needle(serde_json::json!(1)); + assert_eq!(needle, serde_json::json!({"a": [{"x": 1}]})); + let needle = JsonPath::root().needle(serde_json::json!("v")); + assert_eq!(needle, serde_json::json!("v")); + } +} diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs new file mode 100644 index 000000000..944261717 --- /dev/null +++ b/packages/stack-encrypt/src/json/query.rs @@ -0,0 +1,786 @@ +//! The query terms of the JSON index, and how the operators reach it. +//! +//! A containment needle is the **element-nested projection** of the +//! needle's terms: the selector pairs `{s, vs}` of every node, grouped by +//! the array element they sit in. The server projects a stored document the +//! same way from its group tags, and native `jsonb @>` over the two +//! projections is exact containment: each needle element has to fit inside +//! one stored element, which is PostgreSQL's own rule for arrays. The +//! design's "The preferred form" records why no re-check is needed. +use std::fmt; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::derive::{JsonTerms, TermNode}; +use super::path::JsonPath; +use super::{hex, JsonDocument, JsonEntry, JsonOptions, Tag}; +use crate::op::{ + Answers, AtPath, Between, ContainedBy, Contains, DefaultIndex, Eq, Gt, Gte, Input, + LabelledTarget, Lt, Lte, One, Pair, Pathed, QueryTerm, QueryToken, Supports, +}; +use crate::sem::{RangeQuery, TermError}; +use crate::target::{DeclaredContext, Index, Json, Owned, Pending}; +use crate::{Error, KeysetCipher, Label, NonEmpty}; + +/// One selector pair of a projection. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Pair16 { + s: Tag, + vs: Tag, +} + +/// The element-nested projection of a document: the selector pairs that +/// sit outside any array element, and one nested projection per array +/// element that is a container. What the server computes from a stored +/// document's group tags (`json_tree_query`), and what a containment +/// needle is. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct JsonProjection { + sv: Vec, + el: Vec, +} + +impl JsonProjection { + /// `jsonb @>` over two projections: every needle pair is some pair of + /// this projection, and every needle element fits inside some element + /// of it, recursively. What the server computes natively over the two + /// projections; here so a caller can decide containment over documents + /// it holds, with no database. + pub fn contains(&self, needle: &JsonProjection) -> bool { + needle.sv.iter().all(|pair| self.sv.contains(pair)) + && needle + .el + .iter() + .all(|group| self.el.iter().any(|mine| mine.contains(group))) + } +} + +/// The query term of [`Contains`], [`ContainedBy`] and `AtPath`: the +/// element-nested projection of the needle's selectors, and, for a needle +/// whose root is a scalar, the alternative projection that matches a +/// stored root array holding it. Serializes to the `query_json_tree` +/// operand: `{"sv": [{"s": "…", "vs": "…"}, …], "el": [{"sv": …, "el": …}, +/// …], "alt": {…}}`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JsonQuery { + group: JsonProjection, + alt: Option, +} + +impl JsonQuery { + /// The needle of an accessor: the projection of the document that holds + /// the value at the path, with no `alt`. The alternative matches a root + /// array that holds a root scalar, which is containment; at a path, + /// `("$", 1)` asks for the scalar `1`, not for an array holding it. + pub(crate) fn at_path(terms: &JsonTerms) -> Self { + Self { + group: group_of(terms.nodes(), None), + alt: None, + } + } + + pub(crate) fn of_terms(terms: &JsonTerms) -> Self { + let alt = terms.alt().map(|(s, vs)| JsonProjection { + sv: vec![Pair16 { s, vs }], + el: Vec::new(), + }); + Self { + group: group_of(terms.nodes(), None), + alt, + } + } + + /// The needle's projection: what `<@` compares a stored document's + /// projection against. + pub fn projection(&self) -> &JsonProjection { + &self.group + } + + /// Whether a stored document's projection contains this needle, as the + /// server decides `@>`: containment of the needle's projection, or of + /// its alternative for a root scalar against a root array. + pub fn matches(&self, stored: &JsonProjection) -> bool { + stored.contains(&self.group) || self.alt.as_ref().is_some_and(|alt| stored.contains(alt)) + } +} + +/// The group of the nodes that sit directly in `parent` (`None` is the top +/// level), with the container elements in it nested as groups. +fn group_of(nodes: &[TermNode], parent: Option) -> JsonProjection { + let belongs = |index: usize, node: &TermNode| { + if node.container_element { + Some(index) + } else { + node.element + } + }; + JsonProjection { + sv: nodes + .iter() + .enumerate() + .filter(|(i, n)| belongs(*i, n) == parent) + .map(|(_, n)| Pair16 { s: n.s, vs: n.vs }) + .collect(), + el: nodes + .iter() + .enumerate() + .filter(|(_, n)| n.container_element && n.element == parent) + .map(|(i, _)| group_of(nodes, Some(i))) + .collect(), + } +} + +impl JsonDocument { + /// The stored document's projection, from its group tags, as the SQL's + /// `json_tree_query` computes it: what a [`JsonQuery`] is matched + /// against. + pub fn projection(&self) -> JsonProjection { + fn belongs(e: &JsonEntry) -> Option<&Tag> { + e.n.as_ref().or(e.g.as_ref()) + } + fn group(entries: &[JsonEntry], parent: Option<&Tag>) -> JsonProjection { + JsonProjection { + sv: entries + .iter() + .filter(|e| belongs(e) == parent) + .map(|e| Pair16 { s: e.s, vs: e.vs }) + .collect(), + el: entries + .iter() + .filter(|e| e.n.is_some() && e.g.as_ref() == parent) + .map(|e| group(entries, e.n.as_ref())) + .collect(), + } + } + group(&self.entries, None) + } +} + +#[derive(Serialize, Deserialize)] +struct PairWire { + s: String, + vs: String, +} + +#[derive(Serialize, Deserialize)] +struct GroupWire { + sv: Vec, + el: Vec, +} + +#[derive(Serialize, Deserialize)] +struct QueryWire { + #[serde(flatten)] + group: GroupWire, + #[serde(default, skip_serializing_if = "Option::is_none")] + alt: Option, +} + +impl From<&JsonProjection> for GroupWire { + fn from(group: &JsonProjection) -> Self { + Self { + sv: group + .sv + .iter() + .map(|p| PairWire { + s: hex(&p.s), + vs: hex(&p.vs), + }) + .collect(), + el: group.el.iter().map(GroupWire::from).collect(), + } + } +} + +impl TryFrom for JsonProjection { + type Error = super::WireError; + fn try_from(w: GroupWire) -> Result { + let tag = |text: &str| { + Tag::try_from(super::unhex(text)?).map_err(|_| super::WireError("not 16 bytes")) + }; + Ok(Self { + sv: w + .sv + .iter() + .map(|p| { + Ok(Pair16 { + s: tag(&p.s)?, + vs: tag(&p.vs)?, + }) + }) + .collect::>()?, + el: w + .el + .into_iter() + .map(JsonProjection::try_from) + .collect::>()?, + }) + } +} + +impl Serialize for JsonQuery { + fn serialize(&self, s: S) -> Result { + QueryWire { + group: GroupWire::from(&self.group), + alt: self.alt.as_ref().map(GroupWire::from), + } + .serialize(s) + } +} + +impl<'de> Deserialize<'de> for JsonQuery { + fn deserialize>(d: D) -> Result { + let wire = QueryWire::deserialize(d)?; + Ok(Self { + group: JsonProjection::try_from(wire.group).map_err(serde::de::Error::custom)?, + alt: wire + .alt + .map(JsonProjection::try_from) + .transpose() + .map_err(serde::de::Error::custom)?, + }) + } +} + +/// The query term of `AtPath`: the selector of the +/// path and the leaf ordering term of the value, which the SQL compares +/// against the stored leaf's: `col -> $1 >= $2`. Serializes to the +/// `query_json_tree_leaf` operand, `{"s": "…", "op": "…"}`. `Between` +/// sends two as one [`RangeQuery`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JsonLeafQuery { + s: Tag, + op: Vec, +} + +impl JsonLeafQuery { + fn of_leaf(terms: &JsonTerms) -> Self { + // `check` admitted only a number or a string, both of which carry a + // term; an empty term here would be a leaf that cannot be. + let leaf = terms.leaf(); + Self { + s: leaf.map(|n| n.s).unwrap_or_default(), + op: leaf.and_then(|n| n.op.clone()).unwrap_or_default(), + } + } + + /// The path's selector, for `->`. + pub fn selector(&self) -> JsonSelector { + JsonSelector(self.s) + } + + /// The leaf's ordering term, as the `op` of a stored leaf is. + pub fn ordering_term(&self) -> &[u8] { + &self.op + } +} + +#[derive(Serialize, Deserialize)] +struct LeafWire { + s: String, + op: String, +} + +impl Serialize for JsonLeafQuery { + fn serialize(&self, s: S) -> Result { + LeafWire { + s: hex(&self.s), + op: hex(&self.op), + } + .serialize(s) + } +} + +impl<'de> Deserialize<'de> for JsonLeafQuery { + fn deserialize>(d: D) -> Result { + let wire = LeafWire::deserialize(d)?; + let s = super::unhex(&wire.s) + .ok() + .and_then(|b| Tag::try_from(b).ok()) + .ok_or_else(|| serde::de::Error::custom("not a selector"))?; + let op = super::unhex(&wire.op).map_err(serde::de::Error::custom)?; + Ok(Self { s, op }) + } +} + +/// A path selector: what `->` and the path functions take. Prints and +/// serializes as the hex the stored entry carries in `s`. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct JsonSelector(pub(crate) Tag); + +impl JsonSelector { + /// The sixteen bytes. + pub fn as_bytes(&self) -> &Tag { + &self.0 + } +} + +impl fmt::Display for JsonSelector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex(&self.0)) + } +} + +impl fmt::Debug for JsonSelector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "JsonSelector({})", hex(&self.0)) + } +} + +impl Serialize for JsonSelector { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&hex(&self.0)) + } +} + +/// The key of a subtree walk: the token of a positional node, and, when +/// the walked path continues with a wildcard below it, the selector of the +/// whole path the walk gates on. Serializes as `{"t": "…", "s": "…"}`. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct JsonSubtreeKey { + pub(crate) token: Tag, + pub(crate) selector: Option, +} + +impl JsonSubtreeKey { + /// The token, as the hex the walk takes. + pub fn token(&self) -> String { + hex(&self.token) + } + + /// The token's bytes. A token is a capability over the structure of + /// one subtree in every row of the column, for as long as the column + /// lives: send it to the server for a query, and nowhere else. + pub fn token_bytes(&self) -> &Tag { + &self.token + } + + /// The selector the walk gates on, when the path has a wildcard below + /// the token's node. + pub fn selector(&self) -> Option { + self.selector.map(JsonSelector) + } +} + +/// The token is a capability: it prints as the key's shape, not its bytes. +impl fmt::Debug for JsonSubtreeKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("JsonSubtreeKey") + .field("gated", &self.selector.is_some()) + .finish_non_exhaustive() + } +} + +impl Serialize for JsonSubtreeKey { + fn serialize(&self, s: S) -> Result { + #[derive(Serialize)] + struct Wire { + t: String, + #[serde(skip_serializing_if = "Option::is_none")] + s: Option, + } + Wire { + t: hex(&self.token), + s: self.selector.as_ref().map(|t| hex(t)), + } + .serialize(s) + } +} + +/// The terms of an accessor's input: the path it was asked at, and the +/// index's terms of the needle that holds the value there. What +/// [`Input>`](Input) derives, and what the JSON index's +/// [`Supports`] impls build an accessor's query from. +#[derive(Clone, Debug)] +pub struct AtPathTerms { + path: JsonPath, + terms: T, +} + +impl AtPathTerms { + /// The path the query was asked at. + pub fn path(&self) -> &JsonPath { + &self.path + } + + /// The terms of the needle, or of both needles for a pair of bounds. + pub fn terms(&self) -> &T { + &self.terms + } +} + +impl QueryTerm for JsonQuery {} +impl QueryTerm for JsonLeafQuery {} + +// ============================================================================= +// Inputs: a path and a value, or a path and a pair of bounds +// ============================================================================= + +fn parse_path(text: &str) -> Result { + JsonPath::parse(text).map_err(|e| Error::Term(TermError::JsonPath(e))) +} + +/// A path and a value: the input of `AtPath` and the single-bound +/// orderings. The index is declared over the document, so the plaintext +/// is a [`Value`], and the term is derived over the needle that holds the +/// value at the path. +impl Input> for (P, V) +where + P: AsRef + 'static, + V: Into + 'static, +{ + type Plaintext = Value; + type Terms = AtPathTerms; + + fn derive<'a, 'p, I, K>( + self, + index: &I, + keyset: &'p KeysetCipher<'a, K>, + label: Label, + extend: DeclaredContext, + ) -> Pending<'p, AtPathTerms, K> + where + I: Index, + K: 'static, + { + let (path, value) = self; + let path = match parse_path(path.as_ref()) { + Ok(path) => path, + Err(error) => return Pending::failed(keyset, error), + }; + let needle = path.needle(value.into()); + keyset + .run( + index.operation::().under(NonEmpty::from(label)), + needle, + extend, + ) + .map(move |terms| AtPathTerms { path, terms }) + } +} + +/// A path and a pair of bounds: the input of `AtPath`. Both +/// needles derive under the one label, settled together. +impl Input> for (P, (V, V)) +where + P: AsRef + 'static, + V: Into + 'static, +{ + type Plaintext = Value; + type Terms = AtPathTerms<(T, T)>; + + fn derive<'a, 'p, I, K>( + self, + index: &I, + keyset: &'p KeysetCipher<'a, K>, + label: Label, + extend: DeclaredContext, + ) -> Pending<'p, AtPathTerms<(I::Term, I::Term)>, K> + where + I: Index, + K: 'static, + { + let (path, (lo, hi)) = self; + let path = match parse_path(path.as_ref()) { + Ok(path) => path, + Err(error) => return Pending::failed(keyset, error), + }; + let lo = keyset.run( + index + .operation::() + .under(NonEmpty::from(label.clone())), + path.needle(lo.into()), + extend.clone(), + ); + let hi = keyset.run( + index.operation::().under(NonEmpty::from(label)), + path.needle(hi.into()), + extend, + ); + lo.zip(hi).map(move |terms| AtPathTerms { path, terms }) + } +} + +// ============================================================================= +// What the index answers +// ============================================================================= + +/// A document is sealed under the caller's context ([`json_document`] +/// takes a [`CallerContext`](crate::target::CallerContext)), which is the +/// label a plan names, so its terms and a query's derive under the same +/// context and a plan typed with it answers the operators below. +/// +/// [`json_document`]: crate::target::json_document +impl LabelledTarget for JsonDocument {} + +/// Containment, both ways: the needle's projection, which the SQL compares +/// against the stored projection in the direction the operator says. +macro_rules! containment { + ($($op:ident),+ $(,)?) => {$( + impl Supports<$op, Value> for Json { + type Query = JsonQuery; + type Verifier = (); + fn check(&self, _: &Value) -> Result<(), Error> { + Ok(()) + } + fn verifier(&self, _: &Value) {} + fn query((): (), terms: JsonTerms, _: QueryToken) -> JsonQuery { + JsonQuery::of_terms(&terms) + } + } + impl Answers<$op, Value> for JsonDocument { + type Via = Json; + } + impl DefaultIndex for $op { + type Via = Json; + } + )+}; +} +containment!(Contains, ContainedBy); + +/// Equality at a path: the needle that holds the value at the path, +/// projected, with no `alt`. For a scalar, containment of that needle is +/// equality of the value at the path. For an object or an array it would be +/// containment (`{"x": 1}` matching `{"x": 1, "y": 2}`), so a container is +/// refused with [`TermError::JsonValueNotScalar`] when the query is +/// awaited. A position in the path reads as the wildcard, since selectors +/// do not tell positions apart. +impl Supports, (P, V)> for Json +where + P: AsRef + 'static, + V: Into + Clone + 'static, +{ + type Query = JsonQuery; + type Verifier = (); + fn check(&self, (path, value): &(P, V)) -> Result<(), Error> { + let _ = parse_path(path.as_ref())?; + scalar(&value.clone().into()) + } + fn verifier(&self, _: &(P, V)) {} + fn query((): (), terms: AtPathTerms, _: QueryToken) -> JsonQuery { + JsonQuery::at_path(&terms.terms) + } +} +impl Answers, (P, V)> for JsonDocument +where + P: AsRef + 'static, + V: Into + Clone + 'static, +{ + type Via = Json; +} +impl DefaultIndex<(P, V)> for AtPath +where + P: AsRef + 'static, + V: Into + Clone + 'static, +{ + type Via = Json; +} + +/// Whether `value` is a scalar, which equality at a path compares. +fn scalar(value: &Value) -> Result<(), Error> { + match value { + Value::Object(_) | Value::Array(_) => Err(TermError::JsonValueNotScalar.into()), + _ => Ok(()), + } +} + +/// Whether `value` is a leaf the index orders: a number or a string. +fn ordered(value: &Value) -> Result<(), Error> { + match value { + Value::Number(_) | Value::String(_) => Ok(()), + _ => Err(TermError::JsonLeafNotOrdered.into()), + } +} + +/// The orderings at a path: the path's selector and the value's leaf +/// ordering term. Refused, when the query is awaited, for a value that is +/// neither a number nor a string. +macro_rules! orderings { + ($($op:ident),+ $(,)?) => {$( + impl Supports, (P, V)> for Json + where + P: AsRef + 'static, + V: Into + Clone + 'static, + { + type Query = JsonLeafQuery; + type Verifier = (); + fn check(&self, (path, value): &(P, V)) -> Result<(), Error> { + let _ = parse_path(path.as_ref())?; + ordered(&value.clone().into()) + } + fn verifier(&self, _: &(P, V)) {} + fn query((): (), terms: AtPathTerms, _: QueryToken) -> JsonLeafQuery { + JsonLeafQuery::of_leaf(&terms.terms) + } + } + impl Answers, (P, V)> for JsonDocument + where + P: AsRef + 'static, + V: Into + Clone + 'static, + { + type Via = Json; + } + impl DefaultIndex<(P, V)> for AtPath<$op> + where + P: AsRef + 'static, + V: Into + Clone + 'static, + { + type Via = Json; + } + )+}; +} +orderings!(Lt, Lte, Gt, Gte); + +impl Supports, (P, (V, V))> for Json +where + P: AsRef + 'static, + V: Into + Clone + 'static, +{ + type Query = RangeQuery; + type Verifier = (); + fn check(&self, (path, (lo, hi)): &(P, (V, V))) -> Result<(), Error> { + let _ = parse_path(path.as_ref())?; + ordered(&lo.clone().into())?; + ordered(&hi.clone().into()) + } + fn verifier(&self, _: &(P, (V, V))) {} + fn query( + (): (), + terms: AtPathTerms<(JsonTerms, JsonTerms)>, + _: QueryToken, + ) -> RangeQuery { + let (lo, hi) = &terms.terms; + RangeQuery::new(JsonLeafQuery::of_leaf(lo), JsonLeafQuery::of_leaf(hi)) + } +} +impl Answers, (P, (V, V))> for JsonDocument +where + P: AsRef + 'static, + V: Into + Clone + 'static, +{ + type Via = Json; +} +impl DefaultIndex<(P, (V, V))> for AtPath +where + P: AsRef + 'static, + V: Into + Clone + 'static, +{ + type Via = Json; +} + +/// The options a query derives under must be the stored document's; a +/// plan that declares the index carries them, and a query through it takes +/// them from there. +impl Json { + /// The index's layout options. + pub fn options(&self) -> JsonOptions { + self.options + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn a_query_serializes_as_the_needle_operand() { + let pair = |b: u8| Pair16 { + s: [b; 16], + vs: [b + 1; 16], + }; + let query = JsonQuery { + group: JsonProjection { + sv: vec![pair(1)], + el: vec![JsonProjection { + sv: vec![pair(3)], + el: vec![], + }], + }, + alt: None, + }; + let json = serde_json::to_value(&query).unwrap(); + assert_eq!( + json, + json!({ + "sv": [{"s": "01".repeat(16), "vs": "02".repeat(16)}], + "el": [{"sv": [{"s": "03".repeat(16), "vs": "04".repeat(16)}], "el": []}], + }) + ); + let back: JsonQuery = serde_json::from_value(json).unwrap(); + assert_eq!(back, query); + } + + #[test] + fn a_selector_prints_as_its_hex_and_a_subtree_key_hides_its_token() { + let selector = JsonSelector([0xab; 16]); + assert_eq!(selector.to_string(), "ab".repeat(16)); + assert_eq!( + format!("{selector:?}"), + format!("JsonSelector({})", "ab".repeat(16)) + ); + let key = JsonSubtreeKey { + token: [0xcd; 16], + selector: Some([0xab; 16]), + }; + assert_eq!(key.token(), "cd".repeat(16)); + assert_eq!(key.token_bytes(), &[0xcd; 16]); + assert_eq!(key.selector(), Some(selector)); + assert_eq!( + format!("{key:?}"), + "JsonSubtreeKey { gated: true, .. }", + "a token is a capability: it never prints" + ); + } + + #[test] + fn accessor_terms_carry_the_path_they_were_asked_at() { + let terms = AtPathTerms { + path: JsonPath::parse("$.a[*]").unwrap(), + terms: 7u8, + }; + assert_eq!(terms.path(), &JsonPath::parse("$.a[*]").unwrap()); + assert_eq!(terms.terms(), &7); + } + + #[test] + fn the_index_reports_its_options() { + let open = JsonOptions { sealed_root: false }; + assert_eq!(Json::new(open).options(), open); + assert_eq!(Json::default().options(), JsonOptions::default()); + } + + #[test] + fn group_containment_is_jsonbs_rule() { + let pair = |b: u8| Pair16 { + s: [b; 16], + vs: [b; 16], + }; + let element = |pairs: &[u8]| JsonProjection { + sv: pairs.iter().map(|&b| pair(b)).collect(), + el: vec![], + }; + let stored = JsonProjection { + sv: vec![pair(0)], + el: vec![element(&[1, 5]), element(&[7, 2])], + }; + // The issue's example: 1 and 2 sit in different elements. + assert!(!stored.contains(&JsonProjection { + sv: vec![], + el: vec![element(&[1, 2])], + })); + assert!(stored.contains(&JsonProjection { + sv: vec![], + el: vec![element(&[1, 5])], + })); + assert!(stored.contains(&JsonProjection { + sv: vec![pair(0)], + el: vec![element(&[1]), element(&[2])], + })); + assert!(!stored.contains(&JsonProjection { + sv: vec![pair(9)], + el: vec![], + })); + } +} diff --git a/packages/stack-encrypt/src/lib.rs b/packages/stack-encrypt/src/lib.rs index 8afe073e5..001ca4eb9 100644 --- a/packages/stack-encrypt/src/lib.rs +++ b/packages/stack-encrypt/src/lib.rs @@ -274,6 +274,7 @@ mod codes; pub mod descriptor; #[cfg(feature = "dynamic")] pub mod dynamic; +pub mod json; pub mod keyset; pub mod op; pub mod plan; @@ -300,8 +301,8 @@ pub use stack_kms as kms; pub use stack_kms::{diagnostic, ErrorPayload}; pub use target::{ CallerContext, CipherScope, DecryptField, DecryptFrom, DecryptInto, Decryptable, Decryption, - EncryptFrom, EncryptInto, Encrypted, Encryption, Equality, Index, IndexSpec, Indexes, Match, - Ope, Ore, Pending, PendingFuture, Request, Responses, TermSet, + EncryptFrom, EncryptInto, Encrypted, Encryption, Equality, Index, IndexSpec, Indexes, Json, + Match, Ope, Ore, OreBlock, Pending, PendingFuture, Request, Responses, TermSet, }; // Re-export the vitaminc AEAD surface callers need to drive the cipher, so they diff --git a/packages/stack-encrypt/src/op/mod.rs b/packages/stack-encrypt/src/op/mod.rs index d749707cd..005ec2938 100644 --- a/packages/stack-encrypt/src/op/mod.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -6,9 +6,9 @@ //! plan it runs on finds the index that answers it. The operator is a unit //! type passed by value, as an index is (`.via(Ore)`), and it decides what //! the query takes: one value for most, a pair of bounds for `Between`, a -//! path and the leaf operator's input for `AtPath`. No index answers the -//! JSON operators yet: this crate has no JSON index, so a query with one -//! does not compile until that index exists. +//! path and the leaf operator's input for `AtPath`. The JSON operators are +//! answered by the [`Json`](struct@crate::target::Json) index, over a +//! `serde_json::Value`. //! //! ``` //! # async fn example() -> Result<(), stack_encrypt::Error> { @@ -51,8 +51,8 @@ //! operator's input ([`Pathed`](Pathed), where `A` is the leaf's arity). //! [`Input`] says, for the value a query is given, which plaintext the //! index is declared over (`S` for `(S, S)`), so the plan's type is checked -//! against it. No input of a `Pathed` arity exists until the JSON index -//! does. +//! against it. An input of a `Pathed` arity is a path and the leaf's input, +//! `("$.address.city", "Sydney")`, over the JSON index's document. //! //! What the types can see is checked at compile time: a match query on an //! integer, an ordering on bytes, an operator a target type does not answer, @@ -86,10 +86,10 @@ pub(crate) mod stats; pub use stats::{Limit, Stats, DEFAULT_CONFIDENCE, MAX_ROWS}; use crate::sem::{ - DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, OpeTerm, Orderable, OreTerm, - QueryTokens, RangeQuery, + DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, OpeTerm, Orderable, + OreBlockTerm, OreTerm, QueryTokens, RangeQuery, }; -use crate::target::{DeclaredContext, Equality, Index, Match, Ope, Ore, Owned, Pending}; +use crate::target::{DeclaredContext, Equality, Index, Match, Ope, Ore, OreBlock, Owned, Pending}; use crate::{Error, KeysetCipher, Label, NonEmpty, PlanError}; /// A query operator: a unit type naming what a query asks of the server, @@ -155,14 +155,14 @@ impl Operator for Between { } /// JSON containment: rows whose document contains the query's document -/// (`@>`). Exact. Declared by ADR-0009 and answered by no index yet: this -/// crate has no JSON index, so a query with it does not compile until one -/// exists. +/// (`@>`). Exact, as PostgreSQL's own `jsonb @>` is, across array elements +/// included: answered by the [`Json`](struct@crate::target::Json) index with a +/// [`JsonQuery`](crate::json::JsonQuery). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Contains; /// JSON containment the other way: rows whose document is contained by -/// the query's (`<@`). Exact, and answered by no index yet, as +/// the query's (`<@`). Exact, and answered by the JSON index, as /// [`Contains`] is. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct ContainedBy; @@ -177,10 +177,13 @@ pub struct ContainedBy; /// since neither is one selector and value. /// /// Its arity is [`Pathed`](Pathed), so `AtPath` (a path and -/// a value) and `AtPath` (a path and a pair of bounds) take -/// different inputs. No input of it exists yet: the path becomes a -/// selector only a JSON index can derive, and this crate has none, so a -/// query with `AtPath` does not compile until that index exists. +/// a value, `("$.address.city", "Sydney")`) and `AtPath` (a path +/// and a pair of bounds, `("$.score", (10, 20))`) take different inputs. +/// The path is parsed when the query is checked +/// ([`JsonPath`](crate::json::JsonPath)); the JSON index answers `Eq` with +/// containment and the orderings with a leaf term. `AtPath` is +/// declared and answered by nothing: the index derives no match term at a +/// leaf. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct AtPath(pub Op); @@ -238,8 +241,9 @@ pub struct Pair; /// A path and a leaf operator's input, of the leaf's arity `A`: what /// [`AtPath`] takes. `Pathed` is a path and a value, /// `Pathed` a path and a pair of bounds, so an input can derive one -/// term or two as the leaf needs. No [`Input`] of this arity exists yet; -/// it arrives with the JSON index, whose selector the path becomes. +/// term or two as the leaf needs. Its [`Input`]s are `(P, V)` and `(P, (V, +/// V))` for any path text `P` and any `V` that converts into a +/// `serde_json::Value`, over the JSON index's document. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Pathed(PhantomData); @@ -254,13 +258,15 @@ impl LeafArity for Pair {} /// an input of [`Pair`] over `S`. The chain's value type is the input, and /// the plan is checked against the plaintext. /// -/// Those are the two impls; an input of [`Pathed`] arrives with the JSON -/// index. The arity decides how the chain derives terms from the input, -/// which is what [`derive`](Self::derive) does. +/// A `(P, V)` is an input of [`Pathed`] over a `serde_json::Value`, +/// and a `(P, (V, V))` of [`Pathed`]: the path and the value become +/// the document that holds the value at the path, and the JSON index's +/// terms of it. The arity decides how the chain derives terms from the +/// input, which is what [`derive`](Self::derive) does. #[diagnostic::on_unimplemented( message = "`{Self}` is not what a query of arity `{A}` takes", label = "the operator takes another input", - note = "an operator of arity `One` takes one value, `op::Between` (arity `Pair`) takes a pair, `&(lo, hi)`, and `op::AtPath` (arity `Pathed`) takes no input yet: it arrives with the JSON index" + note = "an operator of arity `One` takes one value, `op::Between` (arity `Pair`) takes a pair, `&(lo, hi)`, and `op::AtPath` (arity `Pathed`) takes a path and the leaf's input, `&(\"$.a\", value)` or `&(\"$.a\", (lo, hi))`" )] pub trait Input: Sized + 'static { /// The plaintext the index is declared over. @@ -515,6 +521,7 @@ impl QueryTerm for EqualityTerm {} impl QueryTerm for MatchQuery {} impl QueryTerm for OreTerm {} impl QueryTerm for OpeTerm {} +impl QueryTerm for OreBlockTerm {} impl QueryTerm for RangeQuery {} /// A query term whose answer can hold **false positives and no false @@ -643,11 +650,29 @@ pub trait Supports>: Index { self.check(input)?; Ok(self.verifier(input)) } + /// Why this index cannot answer a query whose terms are derived under + /// `label`, the plan's or the call's, extended by parts the call gave + /// (`.extend(..)`) when `extended` is true: for an index whose query term + /// records the context it was derived under, a context it cannot record. + /// An EQL query value stores the column it queries, so its index refuses + /// a label that names no column, and any extension, which the column + /// cannot hold. Run when the query is awaited, before [`check`](Self::check) + /// and before any keyset is resolved; every index in this crate takes + /// any context and returns `Ok(())`. + fn check_label(&self, _label: &Label, _extended: bool) -> Result<(), Error> { + Ok(()) + } /// The query term, from the verifier and the terms the index derived /// for the input's plaintexts. Only a query chain calls this: /// `QueryToken` cannot be made outside this crate, so no caller can /// pair one query's tokens with another value's filter, or pass a /// stored term off as a query. + /// + /// The token carries the context the terms were derived under + /// ([`QueryToken::label`], [`QueryToken::is_extended`]), for a query + /// term that records it beside its terms. The chain calls this only + /// after [`check_label`](Self::check_label) accepted that context, so a + /// term built from it need not refuse it again. fn query( verifier: Self::Verifier, terms: In::Terms, @@ -655,15 +680,35 @@ pub trait Supports>: Index { ) -> Self::Query; } -/// Proof that [`Supports::query`] is called by a query chain. Only this -/// crate can make one, so an index in another crate can implement -/// [`Supports`] but no caller can turn a term into a query term by hand. +/// Proof that [`Supports::query`] is called by a query chain, and the +/// context the query's terms were derived under. Only this crate can make +/// one, so an index in another crate can implement [`Supports`] but no +/// caller can turn a term into a query term by hand, or name a context its +/// terms were not derived under. #[derive(Debug)] -pub struct QueryToken(()); +pub struct QueryToken { + label: Label, + extended: bool, +} impl QueryToken { - pub(crate) fn new() -> Self { - Self(()) + pub(crate) fn new(label: Label, extended: bool) -> Self { + Self { label, extended } + } + + /// The label the query's terms were derived under, before any + /// extension: the plan's or the call's, the context the write used for + /// the field. It is the whole context exactly when + /// [`is_extended`](Self::is_extended) is false. + pub fn label(&self) -> &Label { + &self.label + } + + /// Whether the call extended the label by parts of its own + /// (`.extend(..)`), so the terms were derived under the label followed + /// by those parts rather than the label alone. + pub fn is_extended(&self) -> bool { + self.extended } } @@ -715,11 +760,14 @@ where } } -/// The ordering operators on the ORE and OPE indexes, over the integer -/// types ([`Ordered`]). The query term for a comparison is the index's own -/// term for the value: this crate's ORE is CLLW, whose ciphertext has no -/// left half to send alone, so what is sent is what is stored, as for an -/// HMAC. `Between` sends both bounds' terms as one `RangeQuery`. +/// The ordering operators on the ORE, OPE and block ORE indexes, over the +/// [`Ordered`] types. The query term for a comparison is the index's own +/// term for the value. CLLW ORE's ciphertext has no left half to send +/// alone, so what is sent is what is stored, as for an HMAC. Block ORE's +/// does, but EQL v4's comparator reads the right half of the operand a +/// btree index or `between` passes second, which is the query, so a block +/// ORE query is the full ciphertext too. `Between` sends both bounds' +/// terms as one `RangeQuery`. macro_rules! orders { ($($index:ident => $term:ident),+ $(,)?) => {$( orders!(@one $index => $term; Lt, Lte, Gt, Gte); @@ -755,10 +803,10 @@ macro_rules! orders { } )+}; } -orders!(Ore => OreTerm, Ope => OpeTerm); +orders!(Ore => OreTerm, Ope => OpeTerm, OreBlock => OreBlockTerm); -/// The ordering operators default to the ORE index; an OPE index is named -/// with `.via(Ope)`. +/// The ordering operators default to the ORE index; an OPE or block ORE +/// index is named with `.via(Ope)` or `.via(OreBlock)`. macro_rules! default_ore { ($($op:ident),+ $(,)?) => {$( impl DefaultIndex for $op { @@ -780,6 +828,10 @@ impl DefaultIndex<(S, S)> for Between { /// dictate, and that choice decides what the query term reveals. A type /// that does not answer an operator has no impl, and the query does not /// compile. +/// +/// To be queried on a typed plan (`Plan::value::().encrypt_into::()`), +/// the type must also implement [`LabelledTarget`], whatever context it is +/// sealed under: an `Answers` impl alone answers no query. #[diagnostic::on_unimplemented( message = "`{Self}` does not answer `{Op}` given `{In}`", label = "this target type does not answer the operator", @@ -791,6 +843,43 @@ pub trait Answers> { type Via: Supports + Default; } +/// A target type whose stored terms were derived under the context a +/// query's label names, so a plan for it +/// (`Plan::value::().encrypt_into::()`) answers the operators its +/// [`Answers`] impls name, with each query derived under the plan's label. +/// +/// Implementing it is a claim about the stored values: a query term derived +/// under the label equals the stored term only if the stored term was +/// derived under the same context. This crate's term targets +/// ([`EqualityTerm`], [`MatchTerms`], [`OreTerm`], [`OpeTerm`], and +/// [`Encrypted`](crate::target::Encrypted) or a ciphertext beside one +/// term) are sealed under the caller's context, which is the label. An EQL +/// v4 type is sealed under the column it stores, which a two-segment label +/// names, and its index refuses any other label +/// ([`Supports::check_label`]). A type sealed under a context no label +/// names, such as an integer tenant id, must not implement it: a query on +/// its plan would match nothing, with no error. +/// +/// A query on such a plan derives through the index alone and never runs +/// the target's own encryption, so the target need not accept the label as +/// its context: an EQL type, sealed under a column identifier, cannot be +/// encrypted through its plan and is queried through it all the same. +#[diagnostic::on_unimplemented( + message = "`{Self}` is not queried under a plan's label", + label = "a typed plan for this target type answers no query", + note = "a target type answers queries on its plan when it implements `op::LabelledTarget`: its stored terms were derived under the context the plan's label names" +)] +pub trait LabelledTarget {} + +impl LabelledTarget for EqualityTerm {} +impl LabelledTarget for MatchTerms {} +impl LabelledTarget for OreTerm {} +impl LabelledTarget for OpeTerm {} +impl LabelledTarget for (crate::StackCipherText, T) {} +/// As the terms beside it are: a ciphertext is sealed under the caller's +/// context, so the terms decide whether the label names theirs. +impl LabelledTarget for crate::target::Encrypted {} + impl Answers for EqualityTerm { type Via = Equality; } @@ -832,7 +921,7 @@ impl, T: Answers> Answers #[diagnostic::on_unimplemented( message = "`{Self}` has no default index given `{In}`", label = "no default index answers this operator here", - note = "`op::Eq` defaults to `Equality`; `op::Matches` defaults to `Match` with the default configuration, which is case-insensitive and over text only; `op::Lt`, `Lte`, `Gt`, `Gte` and `Between` default to `Ore` over an integer, and `Between` takes a pair; name another index, such as `Ope`, with `.via(index)`; no index answers `op::Contains`, `op::ContainedBy` or `op::AtPath` yet, since this crate has no JSON index" + note = "`op::Eq` defaults to `Equality`; `op::Matches` defaults to `Match` with the default configuration, which is case-insensitive and over text only; `op::Lt`, `Lte`, `Gt`, `Gte` and `Between` default to `Ore` over an integer, and `Between` takes a pair; name another index, such as `Ope`, with `.via(index)`; `op::Contains`, `op::ContainedBy` and `op::AtPath` default to `Json` over a `serde_json::Value`" )] pub trait DefaultIndex>: Operator { /// The index. An index is a unit type, so its `Default` is the index. diff --git a/packages/stack-encrypt/src/plan/chain.rs b/packages/stack-encrypt/src/plan/chain.rs index a6842d175..21cf04262 100644 --- a/packages/stack-encrypt/src/plan/chain.rs +++ b/packages/stack-encrypt/src/plan/chain.rs @@ -9,9 +9,9 @@ use uuid::Uuid; use super::build::{FieldPlan, FieldsBuilder, IntoLabel, Opens, PlanContext, Runs}; use super::field_ref::FieldRef; -use super::value::{Indexed, Typed, ValuePlan, ValueShape}; +use super::value::{Indexed, Typed, ValueLayout, ValuePlan, ValueShape}; use super::PlanError; -use crate::op::{Answers, DefaultIndex, Input, Operator, QueryToken, Supports}; +use crate::op::{Answers, DefaultIndex, Input, LabelledTarget, Operator, QueryToken, Supports}; use crate::target::{ ciphertext, Borrowed, CallerContext, DeclaredContext, DecryptField, Decryptable, EncryptFrom, Index, Indexes, Pending, PendingFuture, Select, Whole, @@ -650,7 +650,15 @@ impl<'a, Op: Operator, F: Input, K: 'static> QueryBuilder<'a, Op, F, where P: Chooses, { - self.via(P::Via::default()).on(plan) + let QueryBuilder { common, value, op } = self; + QueryOp::resolve( + common, + value, + op, + (), + |()| P::Via::default(), + |(), call| plan.choose(call), + ) } } @@ -686,22 +694,19 @@ where P: Resolves, { let QueryVia { - mut common, + common, value, op, index, } = self; - let (index, label) = match common.call_context() { - Ok(call) => plan.resolve(index, call), - Err(error) => (index, Err(error)), - }; - QueryOp { + QueryOp::resolve( common, value, - index, - label, op, - } + index, + |index| index, + |index, call| plan.resolve(index, call), + ) } } @@ -711,7 +716,12 @@ where /// /// A field of a fields plan and a one-value plan built `with` indexes /// answer through the operator's [`DefaultIndex`]; a one-value plan for a -/// target type answers through the type's [`Answers`] impl. The plan's +/// target type answers through the type's [`Answers`] impl, when the type +/// is a [`LabelledTarget`], whose stored terms were derived under the +/// context the plan's label names. Such a plan is queried through the +/// indexes the type declares ([`ValueLayout`]) even when the type cannot be +/// encrypted through it, as an EQL type sealed under its column cannot. +/// The plan's /// plaintext is the input's ([`Input::Plaintext`]): `S` for a query given /// an `S`, and for a `Between` given an `(S, S)`. `At` is the position of /// that index in a plan built `with` several, and is inferred. @@ -720,15 +730,21 @@ where label = "no index answers this operator on this plan", note = "a field of a fields plan or a plan built `with` indexes answers an operator through its default index (`op::Eq`: `Equality`; `op::Matches`: `Match` with the default configuration, case-insensitive and over text only; `op::Lt`, `Lte`, `Gt`, `Gte` and `Between`: `Ore` over an integer, `Between` given a pair); a typed plan answers the operators its target type implements `Answers` for; `.via(index)` names an index the plan declares" )] -pub trait Chooses, At>: - Resolves -{ +pub trait Chooses, At>: plan_sealed::Sealed { /// The index that answers `Op` on this plan. type Via: Supports + Default; + /// The index the query runs through on this plan, and the label the + /// write used, or why the plan refuses the query, as + /// [`Resolves::resolve`] says. `call` is the context the call named, if + /// any. + fn choose(&self, call: Option` --> src/op/mod.rs | - | impl Input for (S, S) { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | impl Input for (S, S) { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `(S, S)` implements `Input` + | + ::: src/json/query.rs + | + | / impl Input> for (P, V) + | | where + | | P: AsRef + 'static, + | | V: Into + 'static, + | |_____________________________^ `(P, V)` implements `Input>` +... + | / impl Input> for (P, (V, V)) + | | where + | | P: AsRef + 'static, + | | V: Into + 'static, + | |_____________________________^ `(P, (V, V))` implements `Input>` note: required by a bound in `plan::chain::>::query` --> src/plan/chain.rs | diff --git a/packages/stack-encrypt/tests/ui/op_json_not_answered.rs b/packages/stack-encrypt/tests/ui/op_json_not_answered.rs index 2f208e9fe..7d9f53952 100644 --- a/packages/stack-encrypt/tests/ui/op_json_not_answered.rs +++ b/packages/stack-encrypt/tests/ui/op_json_not_answered.rs @@ -1,6 +1,7 @@ -//! The JSON operators are declared and answered by no index: this crate has -//! no JSON index, so a containment query and a query at a path do not -//! compile on any plan. +//! The JSON operators are answered by the JSON index alone, over a +//! `serde_json::Value`: on a plan over text with an equality index, a +//! containment query and a query at a path do not compile, because the plan +//! declares no index that answers them and its plaintext is not a document. use stack_encrypt::kms::FakeDataKeySource; use stack_encrypt::op; use stack_encrypt::{Equality, Plan, StackCipher}; diff --git a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr index 0d2ea514f..04abbdf46 100644 --- a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr +++ b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr @@ -1,54 +1,73 @@ error[E0277]: `Contains` has no default index given `std::string::String` - --> tests/ui/op_json_not_answered.rs:13:46 + --> tests/ui/op_json_not_answered.rs:14:46 | -13 | let _ = cipher.query(op::Contains, "{}").on(&profile_plan); +14 | let _ = cipher.query(op::Contains, "{}").on(&profile_plan); | ^^ no default index answers this operator here | - = help: the trait `DefaultIndex` is not implemented for `Contains` - = note: `op::Eq` defaults to `Equality`; `op::Matches` defaults to `Match` with the default configuration, which is case-insensitive and over text only; `op::Lt`, `Lte`, `Gt`, `Gte` and `Between` default to `Ore` over an integer, and `Between` takes a pair; name another index, such as `Ope`, with `.via(index)`; no index answers `op::Contains`, `op::ContainedBy` or `op::AtPath` yet, since this crate has no JSON index - = help: the following other types implement trait `DefaultIndex`: - `stack_encrypt::op::Between` implements `DefaultIndex<(S, S)>` - `stack_encrypt::op::Eq` implements `DefaultIndex` - `stack_encrypt::op::Gt` implements `DefaultIndex` - `stack_encrypt::op::Gte` implements `DefaultIndex` - `stack_encrypt::op::Lt` implements `DefaultIndex` - `stack_encrypt::op::Lte` implements `DefaultIndex` - `stack_encrypt::op::Matches<::Case>` implements `DefaultIndex` + = note: `op::Eq` defaults to `Equality`; `op::Matches` defaults to `Match` with the default configuration, which is case-insensitive and over text only; `op::Lt`, `Lte`, `Gt`, `Gte` and `Between` default to `Ore` over an integer, and `Between` takes a pair; name another index, such as `Ope`, with `.via(index)`; `op::Contains`, `op::ContainedBy` and `op::AtPath` default to `Json` over a `serde_json::Value` +help: the trait `DefaultIndex` is not implemented for `Contains` + but trait `DefaultIndex` is implemented for it + --> src/json/query.rs + | + | impl DefaultIndex for $op { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +... + | containment!(Contains, ContainedBy); + | ----------------------------------- in this macro invocation + = help: for that trait implementation, expected `serde_json::value::Value`, found `std::string::String` = note: required for `ValuePlan>` to implement `Chooses` + = note: this error originates in the macro `containment` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0277]: `(&str, &str)` is not what a query of arity `Pathed` takes - --> tests/ui/op_json_not_answered.rs:15:36 +error[E0271]: type mismatch resolving `<(&str, &str) as Input>>::Plaintext == String` + --> tests/ui/op_json_not_answered.rs:17:10 | -15 | .query(op::AtPath(op::Eq), &("$.address.city", "Sydney")) - | ----- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the operator takes another input - | | - | required by a bound introduced by this call +17 | .on(&profile_plan); + | ^^ expected `String`, found `Value` | - = note: an operator of arity `One` takes one value, `op::Between` (arity `Pair`) takes a pair, `&(lo, hi)`, and `op::AtPath` (arity `Pathed`) takes no input yet: it arrives with the JSON index -help: the trait `Input>` is not implemented for `(&str, &str)` - but trait `Input` is implemented for it - --> src/op/mod.rs + = note: required for `ValuePlan>` to implement `Chooses, (&str, &str), _>` + +error[E0277]: the trait bound `stack_encrypt::Equality: Select` is not satisfied + --> tests/ui/op_json_not_answered.rs:17:10 | - | impl Input for (S, S) { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - = help: for that trait implementation, expected `Pair`, found `Pathed` -note: required by a bound in `plan::chain::>::query` - --> src/plan/chain.rs +17 | .on(&profile_plan); + | ^^ the trait `Select` is not implemented for `stack_encrypt::Equality` | - | pub fn query<'a, Op, Q>(&'a self, op: Op, value: &Q) -> QueryBuilder<'a, Op, F, K> - | ----- required by a bound in this associated function -... - | F: Input, - | ^^^^^^^^^^^^^^^^ required by this bound in `plan::chain::>::query` + = help: the following other types implement trait `Select`: + `(A, B)` implements `Select>` + `(A, B)` implements `Select>` + `(A, B, C)` implements `Select>` + `(A, B, C)` implements `Select>` + `(A, B, C)` implements `Select>` + `(A, B, C, D)` implements `Select>` + `(A, B, C, D)` implements `Select>` + `(A, B, C, D)` implements `Select>` + and $N others + = note: required for `ValuePlan>` to implement `Chooses, (&str, &str), _>` -error[E0599]: the method `on` exists for struct `QueryBuilder<'_, AtPath, (&str, &str), FakeDataKeySource>`, but its trait bounds were not satisfied - --> tests/ui/op_json_not_answered.rs:16:10 +error[E0277]: the trait bound `stack_encrypt::Equality: Select` is not satisfied + --> tests/ui/op_json_not_answered.rs:17:13 | -14 | let _ = cipher - | _____________- -15 | | .query(op::AtPath(op::Eq), &("$.address.city", "Sydney")) -16 | | .on(&profile_plan); - | |_________-^^ +17 | .on(&profile_plan); + | -- ^^^^^^^^^^^^^ the trait `Select` is not implemented for `stack_encrypt::Equality` + | | + | required by a bound introduced by this call + | + = help: the following other types implement trait `Select`: + `(A, B)` implements `Select>` + `(A, B)` implements `Select>` + `(A, B, C)` implements `Select>` + `(A, B, C)` implements `Select>` + `(A, B, C)` implements `Select>` + `(A, B, C, D)` implements `Select>` + `(A, B, C, D)` implements `Select>` + `(A, B, C, D)` implements `Select>` + and $N others + = note: required for `ValuePlan>` to implement `Chooses, (&str, &str), _>` +note: required by a bound in `QueryBuilder::<'a, Op, F, K>::on` + --> src/plan/chain.rs | - = note: the following trait bounds were not satisfied: - `(&str, &str): Input>` + | pub fn on(self, plan: &P) -> QueryOp<'a, F, Op, P::Via, K> + | -- required by a bound in this associated function + | where + | P: Chooses, + | ^^^^^^^^^^^^^^^^^^ required by this bound in `QueryBuilder::<'a, Op, F, K>::on` diff --git a/packages/stack-encrypt/tests/ui/op_matches_on_integer.stderr b/packages/stack-encrypt/tests/ui/op_matches_on_integer.stderr index 17e557577..69c72305e 100644 --- a/packages/stack-encrypt/tests/ui/op_matches_on_integer.stderr +++ b/packages/stack-encrypt/tests/ui/op_matches_on_integer.stderr @@ -20,10 +20,10 @@ help: the following other types implement trait `Chooses` | | where | | Op: Operator, | | In: Input, -... | - | | T::Via: Default, - | | Typed: ValueShape, - | |____________________________^ `ValuePlan>` implements `Chooses` + | | S: 'static, + | | T: Answers + LabelledTarget, + | | Typed: ValueLayout, + | |_____________________________^ `ValuePlan>` implements `Chooses` = note: required for `stack_encrypt::op::Matches` to implement `DefaultIndex` = note: required for `ValuePlan>` to implement `Chooses` diff --git a/packages/stack-encrypt/tests/ui/op_target_not_labelled.rs b/packages/stack-encrypt/tests/ui/op_target_not_labelled.rs new file mode 100644 index 000000000..7fd858c9c --- /dev/null +++ b/packages/stack-encrypt/tests/ui/op_target_not_labelled.rs @@ -0,0 +1,41 @@ +//! A target sealed under a context no label names (here an integer tenant +//! id) is not a `LabelledTarget`: a query on its plan would derive under +//! the label and match nothing, so it does not compile, though the type +//! implements `Answers`. (A named index needs the type to accept the label +//! as its context, which it does not either.) +use stack_encrypt::kms::FakeDataKeySource; +use stack_encrypt::sem::EqualityTerm; +use stack_encrypt::target::{EncryptFrom, Encryption, IndexSpec}; +use stack_encrypt::{op, Equality, NonEmpty, Plan, StackCipher}; + +struct ByTenant; + +impl EncryptFrom for ByTenant { + type Context = NonEmpty; + fn encryption<'s, K: 'static>() -> Encryption<'s, String, Self, K, Self::Context> + where + String: 's, + { + >::encryption() + .accepting::>() + .map(|_| ByTenant) + } + fn indexes() -> Vec { + vec![IndexSpec::Equality] + } +} + +impl op::Answers for ByTenant { + type Via = Equality; +} + +async fn query(cipher: &StackCipher) { + let plan = Plan::value::() + .context("users/email") + .encrypt_into::() + .build() + .unwrap(); + let _ = cipher.query(op::Eq, "bob").on(&plan); +} + +fn main() {} diff --git a/packages/stack-encrypt/tests/ui/op_target_not_labelled.stderr b/packages/stack-encrypt/tests/ui/op_target_not_labelled.stderr new file mode 100644 index 000000000..9023a12ed --- /dev/null +++ b/packages/stack-encrypt/tests/ui/op_target_not_labelled.stderr @@ -0,0 +1,21 @@ +error[E0277]: `ByTenant` is not queried under a plan's label + --> tests/ui/op_target_not_labelled.rs:38:41 + | +38 | let _ = cipher.query(op::Eq, "bob").on(&plan); + | ^^ a typed plan for this target type answers no query + | +help: the trait `LabelledTarget` is not implemented for `ByTenant` + --> tests/ui/op_target_not_labelled.rs:11:1 + | +11 | struct ByTenant; + | ^^^^^^^^^^^^^^^ + = note: a target type answers queries on its plan when it implements `op::LabelledTarget`: its stored terms were derived under the context the plan's label names + = help: the following other types implement trait `LabelledTarget`: + (CipherText, T) + Encrypted + EqualityTerm + JsonDocument + MatchTerms + OpeTerm + OreTerm + = note: required for `ValuePlan>` to implement `Chooses` diff --git a/packages/stack-encrypt/tests/ui/struct_field_must_open.stderr b/packages/stack-encrypt/tests/ui/struct_field_must_open.stderr index 59383c476..8335f2e30 100644 --- a/packages/stack-encrypt/tests/ui/struct_field_must_open.stderr +++ b/packages/stack-encrypt/tests/ui/struct_field_must_open.stderr @@ -50,7 +50,7 @@ help: the trait `DecryptField` is not implem `CipherText` implements `DecryptField` `Encrypted` implements `DecryptField` `EqualityTerm` implements `DecryptField` - `MatchTerms` implements `DecryptField` + `JsonDocument` implements `DecryptField` and $N others note: required by a bound in `FieldsBuilder::::encrypt_into` --> src/plan/build.rs