diff --git a/.changeset/eql-v4-block-ore-types.md b/.changeset/eql-v4-block-ore-types.md new file mode 100644 index 000000000..d5cb664f8 --- /dev/null +++ b/.changeset/eql-v4-block-ore-types.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**EQL v4 compares ore-rs's 6-bit block ORE, and Stack Encrypt produces the block ORE types `IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, `DoubleOrdOre`, `TextOrdOre` and `TextSearchOre`.** A v4 `ob` term is one ore-rs ciphertext: the fixed-width scheme for numbers, the chained scheme for text (padded to a power-of-two length bucket of at least 16 bytes). The v4 bundle names its block ORE objects `ore_block_v2` (`eql_v4_internal.ore_block_v2_term`, `compare_ore_block_v2_term`) and compares them with a new comparator that checks each term's shape, raises on malformed or mismatched terms, and runs a constant operation count for terms of one shape. The EQL v3 bundle is unchanged, byte for byte: its `ob` is still cipherstash-client's 8-bit block ORE under `ore_block_256`. See [U-002](../../docs/upgrading/v3.1.md#u-002-the-block-ore-term-is-ore-rss-6-bit-scheme): the comparator refuses the 8-bit terms the previous `eql_v4` bundle compared, so rows or operands holding them must be written again through Stack Encrypt. diff --git a/.changeset/eql-v4-text-search-optional-terms.md b/.changeset/eql-v4-text-search-optional-terms.md index eb7d1e9dc..b9152c801 100644 --- a/.changeset/eql-v4-text-search-optional-terms.md +++ b/.changeset/eql-v4-text-search-optional-terms.md @@ -2,4 +2,4 @@ '@cipherstash/eql': minor --- -**An EQL v4 text search query carries only the terms its operator reads.** `eql_v4.query_text_search` and `eql_v4.query_text_search_ore` now require at least one of their terms (`hm`, `op` or `ob`, `bf`) instead of all of them, so `col = $1` and `col < $1` work on text too short to cut a 3-gram, which has no match term. Each extractor over such a query raises (`invalid_parameter_value`) wherever it reads the operand when the query lacks the term it reads, or holds it as JSON `null` or an empty array, naming the domain, the term and the operators that compare it: `=` without `hm`, `<` without `op`, `@@` without `bf`. So a missing term raises rather than matching no row silently, or every row; the raise is at evaluation, so a generic plan that never reaches the predicate returns no row and no error. Stored `eql_v4_text_search` values still carry every term. The `eql_v3` domains are unchanged: the v3 bundle is byte for byte what it was. With it, **Stack Encrypt produces `TextSearch`** (`eql-bindings`, `stack-encrypt` feature): `TextSearchQuery`'s terms are `Option`s, built per index through `TextSearchEqualityQuery`, `TextSearchOrderQuery` and `TextSearchMatchQuery`, and the stored value carries every term: text too short for a 3-gram stores the empty filter (`"bf": []`, the new `v4::terms::SearchBloomFilter` field type), so `=` and `<` find it and no `@@` query does, where `TextMatch` still refuses it. `TextSearchQuery`'s `Option` terms and `TextSearch`'s `SearchBloomFilter` change the fields of the unreleased `v4` module only; no published `eql-bindings` has it. +**An EQL v4 text search query carries only the terms its operator reads.** `eql_v4.query_text_search` and `eql_v4.query_text_search_ore` now require at least one of their terms (`hm`, `op` or `ob`, `bf`) instead of all of them, so `col = $1` and `col < $1` work on text too short to cut a 3-gram, which has no match term. Each extractor over such a query raises (`invalid_parameter_value`) wherever it reads the operand when the query lacks the term it reads, or holds it as JSON `null` or an empty array, naming the domain, the term and the operators that compare it: `=` without `hm`, `<` without `op`, `@@` without `bf`. So a missing term raises rather than matching no row silently, or every row; the raise is at evaluation, so a generic plan that never reaches the predicate returns no row and no error. Stored `eql_v4_text_search` values still carry every term. The `eql_v3` domains are unchanged: the v3 bundle is byte for byte what it was. With it, **Stack Encrypt produces `TextSearch`** (`eql-bindings`, `stack-encrypt` feature): `TextSearchQuery`'s terms are `Option`s, built per index through `TextSearchEqualityQuery`, `TextSearchOrderQuery` and `TextSearchMatchQuery`, and the stored value carries every term: text too short for a 3-gram stores the empty filter (`"bf": []`, the new `v4::terms::SearchBloomFilter` field type), so `=` and `<` find it and no `@@` query does, where `TextMatch` still refuses it. `TextSearchQuery`'s `Option` terms and `TextSearch`'s `SearchBloomFilter` change the fields of the unreleased `v4` module only; no published `eql-bindings` has it. See [U-001](../../docs/upgrading/v3.1.md#u-001-a-text-search-query-carries-only-the-terms-its-operator-reads) for what a hand-built query must carry, the raise, and the rollback. diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index ead9b325c..e0b1f16be 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -597,6 +597,12 @@ jobs: rustup component add --toolchain "${active_rust_toolchain}" rustfmt clippy mise run test:crates + # The block ORE timing harness's ciphertext generator is a detached + # workspace with its own Cargo.lock that no job builds, so its lock is + # checked here rather than left to drift until someone runs the harness. + - name: Check the timing harness's Cargo.lock + run: cargo metadata --locked --format-version 1 --manifest-path tests/timing/ore_block_v2/ctgen/Cargo.toml > /dev/null + # The three encryption checks are mise tasks rather than raw cargo here, # so `eql-suite-ci.test.mjs` counts them among the cargo tasks a root # workflow must reach, and `eql-workflow-filters.test.mjs` derives the diff --git a/AGENTS.md b/AGENTS.md index 70e6870a5..e25757555 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -695,8 +695,12 @@ exact pin would stop the suite sharing one `cts-common` with it. - **Three Cargo workspaces, not one.** The root workspace (the six stack-* crates and the three node binding crates), protect-ffi's and EQL's. The root `Cargo.toml` excludes the other two, and they pin their own vitaminc. The - three fuzz crates and the two Go guests are detached workspaces too, each - with its own `Cargo.lock`. + three fuzz crates, the two Go guests and the EQL v4 block ORE timing + harness's ciphertext generator + (`packages/eql/tests/timing/ore_block_v2/ctgen`) are detached workspaces + too, each with its own `Cargo.lock`. None of them has a Dependabot entry. + CI builds the generator nowhere, so `test-eql.yml` checks its lock with + `cargo metadata --locked`, and `run.sh` builds it with `--locked`. - **The toolchain is pinned in the root `mise.toml`:** Rust 1.94.1 (the `stack-encrypt` `tests/ui` trybuild snapshots record its diagnostics) and Go 1.26. EQL's `mise.toml` overrides the Rust pin in its folder. diff --git a/Cargo.lock b/Cargo.lock index b14cac340..61991c907 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -350,6 +351,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "brotli" version = "8.0.2" @@ -383,6 +390,12 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -2128,6 +2141,20 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2138,6 +2165,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" @@ -2153,6 +2189,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2201,6 +2258,35 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.6", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3274,6 +3360,7 @@ dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", "stack-auth", @@ -3398,6 +3485,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.114" diff --git a/Cargo.toml b/Cargo.toml index 17bec471a..50f5b3cb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,11 @@ recipher = "=0.3.1" # which Stack Encrypt's order terms are encrypted through. It first ships # in a pre-release, and Cargo resolves a pre-release only when named. cllw-ore = { version = "=0.5.2-rc.1", default-features = false, features = ["vitaminc"] } +# ore-rs's `vitaminc` feature implements the same traits for block ORE: +# `Bit6BlockOre` (fixed width up to ten bytes) and `ChainedBlockOre` +# (variable length), the 6-bit schemes EQL v4's `_ord_ore` domains store. +# Both wire formats are frozen as of 0.9.0. ore-rs feeds only stack-encrypt. +ore-rs = { version = "0.9.0", default-features = false, features = ["vitaminc"] } # External dependencies, with the suite's feature lists. base64 = "0.22.0" diff --git a/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md index e27627aa2..ab2c4fff9 100644 --- a/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md +++ b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md @@ -293,7 +293,9 @@ equality term on the same column still reveals which rows share a timestamp. problem, as it is for a plaintext float column. - **Order:** every scalar kind except `Bool`, under all three schemes, with one exception: block ORE for text, which waits for ore-rs's variable-length - block ORE (see Deferred). + block ORE (see Deferred). *Amended 2026-10-19:* the exception is gone. + ore-rs 0.9.0 released its chained scheme, and Stack Encrypt orders text + with it (see "EQL v4"). - **`Bool` has a ciphertext only.** A keyed hash or an order term over a domain of two values hides nothing: it splits the rows into two groups, and an order term also says which group is `true`. Stack Encrypt stops @@ -472,6 +474,38 @@ without loss, and refuses it otherwise. `-- {{#v4}}` … `-- {{/v4}}`), which the renderer keeps for their own version and drops from the other, so the v3 bundle renders exactly as it did before the block was written. +- *Amended 2026-10-19:* **a v4 `ob` term is ore-rs's 6-bit block ORE.** The + key name is v3's, but the format is not. Stack Encrypt writes one term per + value, the whole ore-rs ciphertext (left and right halves) as one hex + element: the fixed-width scheme (`Bit6BlockOre`, wire header `02 02`) for + the integers and floats, and the chained scheme (`ChainedBlockOre`, `02 + 03`, then the padding policy) for text, padded with NUL bytes to a + power-of-two bucket of at least 16 bytes. v3's `ob` is cipherstash-client's + 8-bit block ORE, up to six terms per text value. The two need different + comparators, so the v4 SQL holds its own (a version block in + `sem/ore_block_256/functions.sql`), and every name built on the block ORE + type is a placeholder too: `{{ore_block}}` renders as `ore_block_256` in + v3 and `ore_block_v2` (ore-rs's wire version 2) in v4, so no v4 object + carries the 8-bit format's name. The v3 bundle renders byte for byte as + before. + + The v4 comparator checks both terms' shape before it compares (header, + block count, exact length, padding policy, every permuted symbol below 64) + and raises, naming the length and never the bytes, on a malformed term, on + terms of different schemes, on fixed-width terms of different block counts + and on chained terms padded under different policies. For terms of one + shape it runs a constant operation count: one statement per block + evaluates both of its comparisons and latches the first difference + arithmetically, and the hash runs exactly once on every path + (`packages/eql/tests/timing/ore_block_v2/` measures it). + The hash is ore-rs's σ-MMO over AES under a fixed public key, which + pgcrypto computes. A query carries the full ciphertext too, not the left + half alone, because a btree index or `between` can pass the stored value + first, and the comparator reads its second operand's right half. + + A comparison of two chained terms reveals the length of their common + prefix in 6-bit blocks, and a chained term reveals its text's length + bucket. Both are ore-rs's documented leakage for the chained scheme. ### Review @@ -563,9 +597,12 @@ through a wildcard. ## Deferred -- **Block ORE for text** waits for ore-rs's chained, variable-length block ORE +- ~~**Block ORE for text** waits for ore-rs's chained, variable-length block ORE to be reviewed and released, then arrives in a minor release of - `vitaminc-ore`. Until then it is an entry in the exceptions list. + `vitaminc-ore`. Until then it is an entry in the exceptions list.~~ + *Amended 2026-10-19:* done. ore-rs 0.9.0's `vitaminc` feature implements + `OrderScheme` for its chained scheme, and Stack Encrypt produces + `TextOrdOre` and `TextSearchOre` with it (see "EQL v4"). - **Block ORE for `Bool`.** Block ORE stored as right ciphertexts only is fully randomised and semantically secure, so a two-value domain leaks nothing through it. `Bool` could support that scheme alone, enforced by a diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index a47152c6a..68a877461 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -116,6 +116,17 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. that orders by ADR-0002's order key: floats as PostgreSQL's `float8` sorts them, text by its fold. No `Between` yet: it needs EQL v4's range query domains. +- **`IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, + `DoubleOrdOre`, `TextOrdOre` and `TextSearchOre`**: a field tagged `encrypt_into=` one of + them gets `Lt`, `Lte`, `Gt` and `Gte` (and on text `Eq`, and on + `TextSearchOre` `Matches`), as the `Ord` types do. They store a block ORE + term (ore-rs's 6-bit schemes), compared by EQL v4's comparator; each + write and each query carries a fresh nonce, so two terms of one value + never share bytes. Text whose fold is longer than 1023 bytes or ends in + NUL is refused, stored or queried. The engine reports their ordering + operators through the index `ore_block` (`record.OreBlock`, + `stashgen.IndexOreBlock`), which a struct reaches only by naming one of + these types: it is not a tag word. - **`TextSearch`**: a `string` field tagged `encrypt_into=TextSearch` gets `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, one column for all of them. The stored value keeps all three terms. Text shorter than three diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 84cb034aa..2603cdf20 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -158,18 +158,18 @@ The engine produces these EQL types today, each from one Go field type: | `encrypt_into` | Go field | |---|---| -| `TextEq`, `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe`, `TextSearch` | `string` | -| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe` | `int16`, or any Go integer type | -| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe` | `int32`, or any Go integer type | -| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe` | `int64`, or any Go integer type | +| `TextEq`, `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe`, `TextOrdOre`, `TextSearch`, `TextSearchOre` | `string` | +| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | `int16`, or any Go integer type | +| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | `int32`, or any Go integer type | +| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | `int64`, or any Go integer type | | `Boolean` | `bool` | -| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe` | `float32` | -| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe` | `float64` | +| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `float32` | +| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `float64` | | `Date`, `DateEq` | `encrypt.Date` | | `Timestamp`, `TimestampEq` | `time.Time` | | `Numeric`, `NumericEq` | `encrypt.Decimal` | -The `Eq` types answer `Eq`; the `Ord` and `OrdOpe` types answer `Lt`, `Lte`, `Gt` and `Gte`, and on text `Eq` too; `TextMatch` answers `Matches`, and `TextSearch` all of `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, each query carrying only the term its operator reads; `Text`, `Smallint`, `Integer`, `Bigint`, `Boolean`, `Real`, `Double`, `Date`, `Timestamp` and `Numeric` are stored and read only, with no query method. +The `Eq` types answer `Eq`; the `Ord`, `OrdOpe` and `OrdOre` types answer `Lt`, `Lte`, `Gt` and `Gte`, and on text `Eq` too; `TextMatch` answers `Matches`, and `TextSearch` and `TextSearchOre` all of `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, each query carrying only the term its operator reads; `Text`, `Smallint`, `Integer`, `Bigint`, `Boolean`, `Real`, `Double`, `Date`, `Timestamp` and `Numeric` are stored and read only, with no query method. The ordered date, timestamp and numeric types are refused: the engine derives no order term for these kinds yet. An ordered type has no `Between` method yet, because a `BETWEEN` operand needs EQL v4's range query domains. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index a9558ab1a..84ab1ddb9 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -142,7 +142,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { t.Fatalf("the generated file does not name the EQL type:\n%s", written) } // A type the engine cannot produce yet: refused by name, nothing written. - unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=TextOrdOre", 1) + unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=DateOrdOre", 1) dir = writeModule(t, map[string]string{"model.go": unproducible}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { @@ -150,7 +150,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // Refused by the generator with the engine's reason, before the engine // is asked: se_targets lists the type as not producible. - if !strings.Contains(stderr.String(), "cannot produce the EQL type TextOrdOre yet") { + if !strings.Contains(stderr.String(), "cannot produce the EQL type DateOrdOre yet") { t.Fatalf("stderr = %q", stderr.String()) } if _, err := os.Stat(filepath.Join(dir, "user_stash.go")); !os.IsNotExist(err) { diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 0b55d690e..d6a82d91c 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -487,25 +487,25 @@ The EQL types the engine produces, and the Go field types each takes: |---|---|---| | `TextEq` | `string` | `=`, `<>` | | `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) | -| `TextOrd`, `TextOrdOpe` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | -| `TextSearch` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY`, `@@` | +| `TextOrd`, `TextOrdOpe`, `TextOrdOre` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `TextSearch`, `TextSearchOre` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY`, `@@` | | `Text` | `string` | none: stored and read only | | `Smallint` | `int16`, or any Go integer type (see below) | none | | `SmallintEq` | the same | `=`, `<>` | -| `SmallintOrd`, `SmallintOrdOpe` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Integer` | `int32`, or any Go integer type (see below) | none | | `IntegerEq` | the same | `=`, `<>` | -| `IntegerOrd`, `IntegerOrdOpe` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Bigint` | `int64`, or any Go integer type | none | | `BigintEq` | the same | `=`, `<>` | -| `BigintOrd`, `BigintOrdOpe` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Boolean` | `bool` | none | | `Real` | `float32` | none | | `RealEq` | `float32` | `=`, `<>` | -| `RealOrd`, `RealOrdOpe` | `float32` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `float32` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Double` | `float64` | none | | `DoubleEq` | `float64` | `=`, `<>` | -| `DoubleOrd`, `DoubleOrdOpe` | `float64` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `float64` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | | `Date` | `encrypt.Date` | none | | `DateEq` | `encrypt.Date` | `=`, `<>` | | `Timestamp` | `time.Time` | none | @@ -560,8 +560,19 @@ answers `Eq` through an equality term only. A text order type also stores `TextEq`'s equality term, and `=` reads that, so `Café` and `cafe` are neither `<` nor `>` each other and not `=` either. Text holding a code point Unicode 16 does not assign has no order term and is refused. There is no -`Between` method yet: a `BETWEEN` needs EQL v4's range query domains. The rest of -the catalog (ORE ordering, ordered dates, timestamps and decimals, JSON) is listed in +`Between` method yet: a `BETWEEN` needs EQL v4's range query domains. + +The `OrdOre` types and `TextSearchOre` order by the same key, through a block +ORE term (ore-rs's 6-bit schemes) in place of the OPE term. Each term +carries a random nonce, so EQL v4's own comparator orders them, not their +bytes: sort with `ORDER BY eql_v4.ord_term_ore(col)`, and index with +`CREATE INDEX ON t USING btree (eql_v4.ord_term_ore(col))`. That index needs +the ORE operator class, which only a superuser install creates; elsewhere +the installer disables these domains, and the `Ord` types are the ones to +use. Text whose fold is longer than 1023 bytes, or ends in NUL, has no block +ORE term and is refused, stored or queried. + +The rest of the catalog (ordered dates, timestamps and decimals, JSON) is listed in `eql.Types` with the reason it is not produced yet, and `stashgen` refuses it with that reason. The guest builds the value: the generated code imports `encrypt/eql`, which embeds the build of the engine that holds the diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index a4ce1228d..3ea4c81eb 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -178,7 +178,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { switch o := record.Output(s); o { // "json" is the SteVec document index, which no plan output // carries. - case record.Equality, record.Match, record.Ore, record.Ope, "json": + case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock, "json": t.Indexes = append(t.Indexes, o) default: return t, fmt.Errorf("unknown index %q", s) @@ -294,10 +294,10 @@ func parseFidelity(s string) (record.Fidelity, error) { } // parseOperatorIndex reads the index that answers an operator: eq, match, -// ore or ope. +// ore, ope or ore_block. func parseOperatorIndex(s string) (record.Output, error) { switch o := record.Output(s); o { - case record.Equality, record.Match, record.Ore, record.Ope: + case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock: return o, nil } return "", fmt.Errorf("unknown index %q", s) diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 1166b43ef..b47e285eb 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -11,51 +11,51 @@ import "database/sql/driver" var Types = []Type{ {Name: "Integer", GoName: "Integer", Family: "integer", Suffix: "", Plaintext: "int32", SQLDomain: "public.eql_v4_integer", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "IntegerEq", GoName: "IntegerEq", Family: "integer", Suffix: "Eq", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_eq", Indexes: []string{"eq"}, Query: "IntegerEqQuery", QuerySQLDomain: "eql_v4.query_integer_eq", Producible: true, Reason: ""}, - {Name: "IntegerOrdOre", GoName: "IntegerOrdOre", Family: "integer", Suffix: "OrdOre", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ore", Indexes: []string{"ore"}, Query: "IntegerOrdOreQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "IntegerOrdOre", GoName: "IntegerOrdOre", Family: "integer", Suffix: "OrdOre", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ore", Indexes: []string{"ore_block"}, Query: "IntegerOrdOreQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ore", Producible: true, Reason: ""}, {Name: "IntegerOrd", GoName: "IntegerOrd", Family: "integer", Suffix: "Ord", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord", Indexes: []string{"ope"}, Query: "IntegerOrdQuery", QuerySQLDomain: "eql_v4.query_integer_ord", Producible: true, Reason: ""}, {Name: "IntegerOrdOpe", GoName: "IntegerOrdOpe", Family: "integer", Suffix: "OrdOpe", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ope", Indexes: []string{"ope"}, Query: "IntegerOrdOpeQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ope", Producible: true, Reason: ""}, {Name: "Smallint", GoName: "Smallint", Family: "smallint", Suffix: "", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "SmallintEq", GoName: "SmallintEq", Family: "smallint", Suffix: "Eq", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_eq", Indexes: []string{"eq"}, Query: "SmallintEqQuery", QuerySQLDomain: "eql_v4.query_smallint_eq", Producible: true, Reason: ""}, - {Name: "SmallintOrdOre", GoName: "SmallintOrdOre", Family: "smallint", Suffix: "OrdOre", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ore", Indexes: []string{"ore"}, Query: "SmallintOrdOreQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "SmallintOrdOre", GoName: "SmallintOrdOre", Family: "smallint", Suffix: "OrdOre", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ore", Indexes: []string{"ore_block"}, Query: "SmallintOrdOreQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ore", Producible: true, Reason: ""}, {Name: "SmallintOrd", GoName: "SmallintOrd", Family: "smallint", Suffix: "Ord", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord", Indexes: []string{"ope"}, Query: "SmallintOrdQuery", QuerySQLDomain: "eql_v4.query_smallint_ord", Producible: true, Reason: ""}, {Name: "SmallintOrdOpe", GoName: "SmallintOrdOpe", Family: "smallint", Suffix: "OrdOpe", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ope", Indexes: []string{"ope"}, Query: "SmallintOrdOpeQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ope", Producible: true, Reason: ""}, {Name: "Bigint", GoName: "Bigint", Family: "bigint", Suffix: "", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "BigintEq", GoName: "BigintEq", Family: "bigint", Suffix: "Eq", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_eq", Indexes: []string{"eq"}, Query: "BigintEqQuery", QuerySQLDomain: "eql_v4.query_bigint_eq", Producible: true, Reason: ""}, - {Name: "BigintOrdOre", GoName: "BigintOrdOre", Family: "bigint", Suffix: "OrdOre", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ore", Indexes: []string{"ore"}, Query: "BigintOrdOreQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "BigintOrdOre", GoName: "BigintOrdOre", Family: "bigint", Suffix: "OrdOre", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ore", Indexes: []string{"ore_block"}, Query: "BigintOrdOreQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ore", Producible: true, Reason: ""}, {Name: "BigintOrd", GoName: "BigintOrd", Family: "bigint", Suffix: "Ord", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord", Indexes: []string{"ope"}, Query: "BigintOrdQuery", QuerySQLDomain: "eql_v4.query_bigint_ord", Producible: true, Reason: ""}, {Name: "BigintOrdOpe", GoName: "BigintOrdOpe", Family: "bigint", Suffix: "OrdOpe", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ope", Indexes: []string{"ope"}, Query: "BigintOrdOpeQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ope", Producible: true, Reason: ""}, {Name: "Date", GoName: "Date", Family: "date", Suffix: "", Plaintext: "date", SQLDomain: "public.eql_v4_date", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "DateEq", GoName: "DateEq", Family: "date", Suffix: "Eq", Plaintext: "date", SQLDomain: "public.eql_v4_date_eq", Indexes: []string{"eq"}, Query: "DateEqQuery", QuerySQLDomain: "eql_v4.query_date_eq", Producible: true, Reason: ""}, - {Name: "DateOrdOre", GoName: "DateOrdOre", Family: "date", Suffix: "OrdOre", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ore", Indexes: []string{"ore"}, Query: "DateOrdOreQuery", QuerySQLDomain: "eql_v4.query_date_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "DateOrdOre", GoName: "DateOrdOre", Family: "date", Suffix: "OrdOre", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ore", Indexes: []string{"ore_block"}, Query: "DateOrdOreQuery", QuerySQLDomain: "eql_v4.query_date_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "DateOrd", GoName: "DateOrd", Family: "date", Suffix: "Ord", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord", Indexes: []string{"ope"}, Query: "DateOrdQuery", QuerySQLDomain: "eql_v4.query_date_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "DateOrdOpe", GoName: "DateOrdOpe", Family: "date", Suffix: "OrdOpe", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ope", Indexes: []string{"ope"}, Query: "DateOrdOpeQuery", QuerySQLDomain: "eql_v4.query_date_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "Timestamp", GoName: "Timestamp", Family: "timestamp", Suffix: "", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "TimestampEq", GoName: "TimestampEq", Family: "timestamp", Suffix: "Eq", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_eq", Indexes: []string{"eq"}, Query: "TimestampEqQuery", QuerySQLDomain: "eql_v4.query_timestamp_eq", Producible: true, Reason: ""}, - {Name: "TimestampOrdOre", GoName: "TimestampOrdOre", Family: "timestamp", Suffix: "OrdOre", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ore", Indexes: []string{"ore"}, Query: "TimestampOrdOreQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "TimestampOrdOre", GoName: "TimestampOrdOre", Family: "timestamp", Suffix: "OrdOre", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ore", Indexes: []string{"ore_block"}, Query: "TimestampOrdOreQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "TimestampOrd", GoName: "TimestampOrd", Family: "timestamp", Suffix: "Ord", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord", Indexes: []string{"ope"}, Query: "TimestampOrdQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "TimestampOrdOpe", GoName: "TimestampOrdOpe", Family: "timestamp", Suffix: "OrdOpe", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ope", Indexes: []string{"ope"}, Query: "TimestampOrdOpeQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "Numeric", GoName: "Numeric", Family: "numeric", Suffix: "", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "NumericEq", GoName: "NumericEq", Family: "numeric", Suffix: "Eq", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_eq", Indexes: []string{"eq"}, Query: "NumericEqQuery", QuerySQLDomain: "eql_v4.query_numeric_eq", Producible: true, Reason: ""}, - {Name: "NumericOrdOre", GoName: "NumericOrdOre", Family: "numeric", Suffix: "OrdOre", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ore", Indexes: []string{"ore"}, Query: "NumericOrdOreQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "NumericOrdOre", GoName: "NumericOrdOre", Family: "numeric", Suffix: "OrdOre", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ore", Indexes: []string{"ore_block"}, Query: "NumericOrdOreQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "NumericOrd", GoName: "NumericOrd", Family: "numeric", Suffix: "Ord", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord", Indexes: []string{"ope"}, Query: "NumericOrdQuery", QuerySQLDomain: "eql_v4.query_numeric_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "NumericOrdOpe", GoName: "NumericOrdOpe", Family: "numeric", Suffix: "OrdOpe", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ope", Indexes: []string{"ope"}, Query: "NumericOrdOpeQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, {Name: "Text", GoName: "Text", Family: "text", Suffix: "", Plaintext: "string", SQLDomain: "public.eql_v4_text", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "TextEq", GoName: "TextEq", Family: "text", Suffix: "Eq", Plaintext: "string", SQLDomain: "public.eql_v4_text_eq", Indexes: []string{"eq"}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true, Reason: ""}, {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v4_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v4.query_text_match", Producible: true, Reason: ""}, - {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ore", Indexes: []string{"eq", "ore"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v4.query_text_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ore", Indexes: []string{"eq", "ore_block"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v4.query_text_ord_ore", Producible: true, Reason: ""}, {Name: "TextOrd", GoName: "TextOrd", Family: "text", Suffix: "Ord", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord", Indexes: []string{"eq", "ope"}, Query: "TextOrdQuery", QuerySQLDomain: "eql_v4.query_text_ord", Producible: true, Reason: ""}, {Name: "TextOrdOpe", GoName: "TextOrdOpe", Family: "text", Suffix: "OrdOpe", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ope", Indexes: []string{"eq", "ope"}, Query: "TextOrdOpeQuery", QuerySQLDomain: "eql_v4.query_text_ord_ope", Producible: true, Reason: ""}, - {Name: "TextSearchOre", GoName: "TextSearchOre", Family: "text", Suffix: "SearchOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_search_ore", Indexes: []string{"eq", "ore", "match"}, Query: "TextSearchOreQuery", QuerySQLDomain: "eql_v4.query_text_search_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "TextSearchOre", GoName: "TextSearchOre", Family: "text", Suffix: "SearchOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_search_ore", Indexes: []string{"eq", "ore_block", "match"}, Query: "TextSearchOreQuery", QuerySQLDomain: "eql_v4.query_text_search_ore", Producible: true, Reason: ""}, {Name: "TextSearch", GoName: "TextSearch", Family: "text", Suffix: "Search", Plaintext: "string", SQLDomain: "public.eql_v4_text_search", Indexes: []string{"eq", "ope", "match"}, Query: "TextSearchQuery", QuerySQLDomain: "eql_v4.query_text_search", Producible: true, Reason: ""}, {Name: "Boolean", GoName: "Boolean", Family: "boolean", Suffix: "", Plaintext: "bool", SQLDomain: "public.eql_v4_boolean", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "Real", GoName: "Real", Family: "real", Suffix: "", Plaintext: "float32", SQLDomain: "public.eql_v4_real", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "RealEq", GoName: "RealEq", Family: "real", Suffix: "Eq", Plaintext: "float32", SQLDomain: "public.eql_v4_real_eq", Indexes: []string{"eq"}, Query: "RealEqQuery", QuerySQLDomain: "eql_v4.query_real_eq", Producible: true, Reason: ""}, - {Name: "RealOrdOre", GoName: "RealOrdOre", Family: "real", Suffix: "OrdOre", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ore", Indexes: []string{"ore"}, Query: "RealOrdOreQuery", QuerySQLDomain: "eql_v4.query_real_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "RealOrdOre", GoName: "RealOrdOre", Family: "real", Suffix: "OrdOre", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ore", Indexes: []string{"ore_block"}, Query: "RealOrdOreQuery", QuerySQLDomain: "eql_v4.query_real_ord_ore", Producible: true, Reason: ""}, {Name: "RealOrd", GoName: "RealOrd", Family: "real", Suffix: "Ord", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord", Indexes: []string{"ope"}, Query: "RealOrdQuery", QuerySQLDomain: "eql_v4.query_real_ord", Producible: true, Reason: ""}, {Name: "RealOrdOpe", GoName: "RealOrdOpe", Family: "real", Suffix: "OrdOpe", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ope", Indexes: []string{"ope"}, Query: "RealOrdOpeQuery", QuerySQLDomain: "eql_v4.query_real_ord_ope", Producible: true, Reason: ""}, {Name: "Double", GoName: "Double", Family: "double", Suffix: "", Plaintext: "float64", SQLDomain: "public.eql_v4_double", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "DoubleEq", GoName: "DoubleEq", Family: "double", Suffix: "Eq", Plaintext: "float64", SQLDomain: "public.eql_v4_double_eq", Indexes: []string{"eq"}, Query: "DoubleEqQuery", QuerySQLDomain: "eql_v4.query_double_eq", Producible: true, Reason: ""}, - {Name: "DoubleOrdOre", GoName: "DoubleOrdOre", Family: "double", Suffix: "OrdOre", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ore", Indexes: []string{"ore"}, Query: "DoubleOrdOreQuery", QuerySQLDomain: "eql_v4.query_double_ord_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, + {Name: "DoubleOrdOre", GoName: "DoubleOrdOre", Family: "double", Suffix: "OrdOre", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ore", Indexes: []string{"ore_block"}, Query: "DoubleOrdOreQuery", QuerySQLDomain: "eql_v4.query_double_ord_ore", Producible: true, Reason: ""}, {Name: "DoubleOrd", GoName: "DoubleOrd", Family: "double", Suffix: "Ord", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord", Indexes: []string{"ope"}, Query: "DoubleOrdQuery", QuerySQLDomain: "eql_v4.query_double_ord", Producible: true, Reason: ""}, {Name: "DoubleOrdOpe", GoName: "DoubleOrdOpe", Family: "double", Suffix: "OrdOpe", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ope", Indexes: []string{"ope"}, Query: "DoubleOrdOpeQuery", QuerySQLDomain: "eql_v4.query_double_ord_ope", Producible: true, Reason: ""}, {Name: "SteVecDocument", GoName: "SteVecDocument", Family: "json", Suffix: "Search", Plaintext: "", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "SteVecQuery", QuerySQLDomain: "eql_v4.query_json", Producible: false, Reason: "the JSON index is a new operation in the engine (#1060)"}, @@ -133,8 +133,8 @@ func (v *IntegerEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrdOre is the EQL type stored in public.eql_v4_integer_ord_ore: the integer family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// IntegerOrdOre is the EQL type stored in public.eql_v4_integer_ord_ore: the integer family with the ore_block index. +// The engine produces it from an int32. type IntegerOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -345,8 +345,8 @@ func (v *SmallintEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrdOre is the EQL type stored in public.eql_v4_smallint_ord_ore: the smallint family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// SmallintOrdOre is the EQL type stored in public.eql_v4_smallint_ord_ore: the smallint family with the ore_block index. +// The engine produces it from an int16. type SmallintOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -557,8 +557,8 @@ func (v *BigintEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrdOre is the EQL type stored in public.eql_v4_bigint_ord_ore: the bigint family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// BigintOrdOre is the EQL type stored in public.eql_v4_bigint_ord_ore: the bigint family with the ore_block index. +// The engine produces it from an int64. type BigintOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -769,7 +769,7 @@ func (v *DateEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrdOre is the EQL type stored in public.eql_v4_date_ord_ore: the date family with the ore index. +// DateOrdOre is the EQL type stored in public.eql_v4_date_ord_ore: the date family with the ore_block index. // The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type DateOrdOre []byte @@ -981,7 +981,7 @@ func (v *TimestampEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrdOre is the EQL type stored in public.eql_v4_timestamp_ord_ore: the timestamp family with the ore index. +// TimestampOrdOre is the EQL type stored in public.eql_v4_timestamp_ord_ore: the timestamp family with the ore_block index. // The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type TimestampOrdOre []byte @@ -1193,7 +1193,7 @@ func (v *NumericEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrdOre is the EQL type stored in public.eql_v4_numeric_ord_ore: the numeric family with the ore index. +// NumericOrdOre is the EQL type stored in public.eql_v4_numeric_ord_ore: the numeric family with the ore_block index. // The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type NumericOrdOre []byte @@ -1452,8 +1452,8 @@ func (v *TextMatchQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrdOre is the EQL type stored in public.eql_v4_text_ord_ore: the text family with the eq and ore indexes. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// TextOrdOre is the EQL type stored in public.eql_v4_text_ord_ore: the text family with the eq and ore_block indexes. +// The engine produces it from a string. type TextOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1593,8 +1593,8 @@ func (v *TextOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextSearchOre is the EQL type stored in public.eql_v4_text_search_ore: the text family with the eq and ore and match indexes. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// TextSearchOre is the EQL type stored in public.eql_v4_text_search_ore: the text family with the eq and ore_block and match indexes. +// The engine produces it from a string. type TextSearchOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1782,8 +1782,8 @@ func (v *RealEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrdOre is the EQL type stored in public.eql_v4_real_ord_ore: the real family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// RealOrdOre is the EQL type stored in public.eql_v4_real_ord_ore: the real family with the ore_block index. +// The engine produces it from a float32. type RealOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1994,8 +1994,8 @@ func (v *DoubleEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrdOre is the EQL type stored in public.eql_v4_double_ord_ore: the double family with the ore index. -// The engine cannot produce it yet: EQL stores block ORE terms and the engine derives CLLW ORE (#1132). +// DoubleOrdOre is the EQL type stored in public.eql_v4_double_ord_ore: the double family with the ore_block index. +// The engine produces it from a float64. type DoubleOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2206,7 +2206,7 @@ func (v *JSON) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerRangeOreQuery is a range query value over the ore index of the integer family: both bounds of one BETWEEN, +// IntegerRangeOreQuery is a range query value over the ore_block index of the integer family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_integer_range_ore takes in eql_v4.between. It ranges over IntegerOrdOre. type IntegerRangeOreQuery []byte @@ -2254,7 +2254,7 @@ func (v *IntegerRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintRangeOreQuery is a range query value over the ore index of the smallint family: both bounds of one BETWEEN, +// SmallintRangeOreQuery is a range query value over the ore_block index of the smallint family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_smallint_range_ore takes in eql_v4.between. It ranges over SmallintOrdOre. type SmallintRangeOreQuery []byte @@ -2302,7 +2302,7 @@ func (v *SmallintRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintRangeOreQuery is a range query value over the ore index of the bigint family: both bounds of one BETWEEN, +// BigintRangeOreQuery is a range query value over the ore_block index of the bigint family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_bigint_range_ore takes in eql_v4.between. It ranges over BigintOrdOre. type BigintRangeOreQuery []byte @@ -2350,7 +2350,7 @@ func (v *BigintRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateRangeOreQuery is a range query value over the ore index of the date family: both bounds of one BETWEEN, +// DateRangeOreQuery is a range query value over the ore_block index of the date family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_date_range_ore takes in eql_v4.between. It ranges over DateOrdOre. type DateRangeOreQuery []byte @@ -2398,7 +2398,7 @@ func (v *DateRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampRangeOreQuery is a range query value over the ore index of the timestamp family: both bounds of one BETWEEN, +// TimestampRangeOreQuery is a range query value over the ore_block index of the timestamp family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_timestamp_range_ore takes in eql_v4.between. It ranges over TimestampOrdOre. type TimestampRangeOreQuery []byte @@ -2446,7 +2446,7 @@ func (v *TimestampRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericRangeOreQuery is a range query value over the ore index of the numeric family: both bounds of one BETWEEN, +// NumericRangeOreQuery is a range query value over the ore_block index of the numeric family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_numeric_range_ore takes in eql_v4.between. It ranges over NumericOrdOre. type NumericRangeOreQuery []byte @@ -2494,7 +2494,7 @@ func (v *NumericRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextRangeOreQuery is a range query value over the ore index of the text family: both bounds of one BETWEEN, +// TextRangeOreQuery is a range query value over the ore_block index of the text family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_text_range_ore takes in eql_v4.between. It ranges over TextOrdOre, TextSearchOre. type TextRangeOreQuery []byte @@ -2542,7 +2542,7 @@ func (v *TextRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealRangeOreQuery is a range query value over the ore index of the real family: both bounds of one BETWEEN, +// RealRangeOreQuery is a range query value over the ore_block index of the real family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_real_range_ore takes in eql_v4.between. It ranges over RealOrdOre. type RealRangeOreQuery []byte @@ -2590,7 +2590,7 @@ func (v *RealRangeOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleRangeOreQuery is a range query value over the ore index of the double family: both bounds of one BETWEEN, +// DoubleRangeOreQuery is a range query value over the ore_block index of the double family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_double_range_ore takes in eql_v4.between. It ranges over DoubleOrdOre. type DoubleRangeOreQuery []byte diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index 6a0563ffd..94067930a 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -20,14 +20,14 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { } } want := []string{ - "Integer", "IntegerEq", "IntegerOrd", "IntegerOrdOpe", - "Smallint", "SmallintEq", "SmallintOrd", "SmallintOrdOpe", - "Bigint", "BigintEq", "BigintOrd", "BigintOrdOpe", + "Integer", "IntegerEq", "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", + "Smallint", "SmallintEq", "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", + "Bigint", "BigintEq", "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", "DateEq", "Timestamp", "TimestampEq", "Numeric", "NumericEq", - "Text", "TextEq", "TextMatch", "TextOrd", "TextOrdOpe", "TextSearch", + "Text", "TextEq", "TextMatch", "TextOrdOre", "TextOrd", "TextOrdOpe", "TextSearchOre", "TextSearch", "Boolean", - "Real", "RealEq", "RealOrd", "RealOrdOpe", - "Double", "DoubleEq", "DoubleOrd", "DoubleOrdOpe", + "Real", "RealEq", "RealOrdOre", "RealOrd", "RealOrdOpe", + "Double", "DoubleEq", "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", } if len(producible) != len(want) { t.Fatalf("producible = %v, want %v", producible, want) @@ -37,7 +37,7 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { t.Fatalf("producible = %v, want %v", producible, want) } } - for name, plaintext := range map[string]string{"Integer": "int32", "IntegerEq": "int32", "IntegerOrd": "int32", "Smallint": "int16", "SmallintEq": "int16", "SmallintOrdOpe": "int16", "Bigint": "int64", "BigintEq": "int64", "BigintOrdOpe": "int64", "Boolean": "bool", "Real": "float32", "RealEq": "float32", "RealOrd": "float32", "Double": "float64", "DoubleEq": "float64", "DoubleOrdOpe": "float64", "TextOrd": "string", "Date": "date", "DateEq": "date", "Timestamp": "timestamp", "TimestampEq": "timestamp", "Numeric": "decimal", "NumericEq": "decimal"} { + for name, plaintext := range map[string]string{"Integer": "int32", "IntegerEq": "int32", "IntegerOrdOre": "int32", "TextOrdOre": "string", "IntegerOrd": "int32", "Smallint": "int16", "SmallintEq": "int16", "SmallintOrdOre": "int16", "SmallintOrdOpe": "int16", "Bigint": "int64", "BigintEq": "int64", "BigintOrdOpe": "int64", "Boolean": "bool", "Real": "float32", "RealEq": "float32", "RealOrd": "float32", "Double": "float64", "DoubleEq": "float64", "DoubleOrdOpe": "float64", "TextOrd": "string", "Date": "date", "DateEq": "date", "Timestamp": "timestamp", "TimestampEq": "timestamp", "Numeric": "decimal", "NumericEq": "decimal"} { if typ, ok := eql.Lookup(name); !ok || typ.Plaintext != plaintext { t.Errorf("%s: Plaintext = %q, want %q", name, typ.Plaintext, plaintext) } diff --git a/languages/golang/encrypt/eql_test.go b/languages/golang/encrypt/eql_test.go index a1cd3fb19..de73e01bd 100644 --- a/languages/golang/encrypt/eql_test.go +++ b/languages/golang/encrypt/eql_test.go @@ -315,7 +315,7 @@ func TestTextEqIsRefusedByTheBuildWithoutEQLTypes(t *testing.T) { if err := eqlChecker.Check(ctx, plan); err != nil { t.Fatalf("the eql build refuses TextEq: %v", err) } - for _, name := range []string{"TextOrdOre", "Nope"} { + for _, name := range []string{"DateOrdOre", "Nope"} { refused := &record.Plan{Context: []string{"users"}, Fields: []record.Field{{Name: "email", Kind: record.String, Target: name}}} if err := eqlChecker.Check(ctx, refused); !errors.Is(err, encrypt.ErrEncoding) { t.Errorf("%s: Check = %v, want ErrEncoding", name, err) diff --git a/languages/golang/encrypt/guest/Cargo.lock b/languages/golang/encrypt/guest/Cargo.lock index a5bb1941d..f9bc1224b 100644 --- a/languages/golang/encrypt/guest/Cargo.lock +++ b/languages/golang/encrypt/guest/Cargo.lock @@ -21,6 +21,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -218,6 +219,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "bumpalo" version = "3.20.3" @@ -230,6 +237,12 @@ version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -1344,6 +1357,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1354,6 +1381,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1369,6 +1405,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1400,6 +1457,35 @@ dependencies = [ "libc", ] +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.8", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -2079,6 +2165,7 @@ dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", "stack-encrypt-derive", @@ -2200,6 +2287,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.119" diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 450e655d3..c30b5430b 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -810,7 +810,7 @@ mod tests { assert_eq!(plan_check(&plan), Ok(()), "the eql build runs TextEq"); // A type the engine cannot produce, and one that does not exist, // are refused at plan_check. - for name in ["TextOrdOre", "Nope"] { + for name in ["DateOrdOre", "Nope"] { let refused = encoded(object(vec![( "email", object(vec![ diff --git a/languages/golang/encrypt/guest/src/status.rs b/languages/golang/encrypt/guest/src/status.rs index 2215ac536..07da13b1a 100644 --- a/languages/golang/encrypt/guest/src/status.rs +++ b/languages/golang/encrypt/guest/src/status.rs @@ -432,8 +432,8 @@ mod tests { name: "Nope".into(), }, TargetError::Unproducible { - name: "TextOrdOre".into(), - reason: "block ORE".into(), + name: "DateOrdOre".into(), + reason: "no kind for this family".into(), }, TargetError::Extended { name: "email".into(), diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index 2095e3c1d..816cfce0e 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -193,14 +193,17 @@ mod tests { [ "Integer", "IntegerEq", + "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", "Smallint", "SmallintEq", + "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", "Bigint", "BigintEq", + "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", @@ -212,23 +215,27 @@ mod tests { "Text", "TextEq", "TextMatch", + "TextOrdOre", "TextOrd", "TextOrdOpe", + "TextSearchOre", "TextSearch", "Boolean", "Real", "RealEq", + "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", + "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe" ], "the engine's producible types, in catalog order" ); assert!(matches!( - resolver().resolve("TextOrdOre"), + resolver().resolve("DateOrdOre"), Err(TargetError::Unproducible { .. }) )); assert!(matches!( diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 6bf48fd8c..c386c6e37 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1365,14 +1365,17 @@ fn targets_lists_what_the_build_holds() { [ "Integer", "IntegerEq", + "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", "Smallint", "SmallintEq", + "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", "Bigint", "BigintEq", + "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", @@ -1384,16 +1387,20 @@ fn targets_lists_what_the_build_holds() { "Text", "TextEq", "TextMatch", + "TextOrdOre", "TextOrd", "TextOrdOpe", + "TextSearchOre", "TextSearch", "Boolean", "Real", "RealEq", + "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", + "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe" ], @@ -1789,13 +1796,20 @@ fn the_catalog_lists_each_types_operators() { ("index", index), ] }; - let orderings = || ["lt", "lte", "gt", "gte"].map(|op| exact(op, "ope")); + // OPE for Ord and OrdOpe, block ORE for OrdOre and SearchOre. + let order_index = if name.ends_with("Ore") { + "ore_block" + } else { + "ope" + }; + let orderings = || ["lt", "lte", "gt", "gte"].map(|op| exact(op, order_index)); match name.as_str() { - "IntegerOrd" | "IntegerOrdOpe" | "SmallintOrd" | "SmallintOrdOpe" | "BigintOrd" - | "BigintOrdOpe" | "RealOrd" | "RealOrdOpe" | "DoubleOrd" | "DoubleOrdOpe" => { + "IntegerOrd" | "IntegerOrdOpe" | "IntegerOrdOre" | "SmallintOrd" | "SmallintOrdOpe" + | "SmallintOrdOre" | "BigintOrd" | "BigintOrdOpe" | "BigintOrdOre" | "RealOrd" + | "RealOrdOpe" | "RealOrdOre" | "DoubleOrd" | "DoubleOrdOpe" | "DoubleOrdOre" => { assert_eq!(listed, orderings().to_vec(), "{name}"); } - "TextOrd" | "TextOrdOpe" => { + "TextOrd" | "TextOrdOpe" | "TextOrdOre" => { let mut want = vec![exact("eq", "eq")]; want.extend(orderings()); assert_eq!(listed, want, "{name}"); @@ -1815,7 +1829,7 @@ fn the_catalog_lists_each_types_operators() { ]), // Each operator through its own index, whose term alone the // query carries. - "TextSearch" => { + "TextSearch" | "TextSearchOre" => { let mut want = vec![exact("eq", "eq")]; want.extend(orderings()); want.push(vec![ diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index 9ba2f0fe5..92559a6f5 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -382,10 +382,10 @@ func TestVerifyAndTargetLimitNeedNoCipher(t *testing.T) { } // The catalog reports, per EQL type, the operators the engine answers: -// TextEq answers eq, exactly, through its equality index, and an OPE-ordered -// type the four orderings of one value through its ope index (Between takes -// a pair, which a type's one query value does not carry), and eq too when it -// also stores an equality term. +// TextEq answers eq, exactly, through its equality index, and an ordered type +// the four orderings of one value through its order index, ope or ore_block +// (Between takes a pair, which a type's one query value does not carry), and +// eq too when it also stores an equality term. func TestTheCatalogListsEachTypesOperators(t *testing.T) { wasm, err := encrypt.EmbeddedGuest() if errors.Is(err, encrypt.ErrGuestNotBuilt) { @@ -407,21 +407,39 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if len(targets) == 0 { t.Skip("this build holds no EQL types: import encrypt/eql") } + families := []string{"Integer", "Smallint", "Bigint", "Real", "Double", "Text"} + // Stored and compared for equality only today: the engine derives no + // order term for these kinds, so their ordered types are refused. + equalityOnly := []string{"Date", "Timestamp", "Numeric"} + // The index a type answers the orderings through: OPE for Ord and + // OrdOpe, block ORE for OrdOre. + orderIndex := map[string]record.Output{} + for _, family := range families { + orderIndex[family+"Ord"] = record.Ope + orderIndex[family+"OrdOpe"] = record.Ope + orderIndex[family+"OrdOre"] = record.OreBlock + } + orderings := func(index record.Output) []record.Operator { + var ops []record.Operator + for _, name := range []string{"lt", "lte", "gt", "gte"} { + ops = append(ops, record.Operator{Name: name, Fidelity: record.Exact, Index: index}) + } + return ops + } for _, target := range targets { + index, ordered := orderIndex[target.Name] switch { - case slices.Contains([]string{"TextEq", "IntegerEq", "SmallintEq", "BigintEq", "RealEq", "DoubleEq", "DateEq", "TimestampEq", "NumericEq"}, target.Name): + case strings.HasSuffix(target.Name, "Eq") && slices.Contains(slices.Concat(families, equalityOnly), strings.TrimSuffix(target.Name, "Eq")): want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} if len(target.Operators) != 1 || target.Operators[0] != want[0] { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } - case slices.Contains([]string{"IntegerOrd", "IntegerOrdOpe", "SmallintOrd", "SmallintOrdOpe", "BigintOrd", "BigintOrdOpe", "RealOrd", "RealOrdOpe", "DoubleOrd", "DoubleOrdOpe", "TextOrd", "TextOrdOpe"}, target.Name): + case ordered: var want []record.Operator if strings.HasPrefix(target.Name, "Text") { want = append(want, record.Operator{Name: "eq", Fidelity: record.Exact, Index: record.Equality}) } - for _, name := range []string{"lt", "lte", "gt", "gte"} { - want = append(want, record.Operator{Name: name, Fidelity: record.Exact, Index: record.Ope}) - } + want = append(want, orderings(index)...) if !slices.Equal(target.Operators, want) { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } @@ -430,15 +448,17 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if len(target.Operators) != 1 || target.Operators[0] != want { t.Fatalf("TextMatch answers %+v, want %+v", target.Operators, want) } - case target.Name == "TextSearch": + case target.Name == "TextSearch" || target.Name == "TextSearchOre": // Each through its own index: the query carries that term alone. - want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} - for _, name := range []string{"lt", "lte", "gt", "gte"} { - want = append(want, record.Operator{Name: name, Fidelity: record.Exact, Index: record.Ope}) + index := record.Ope + if target.Name == "TextSearchOre" { + index = record.OreBlock } + want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} + want = append(want, orderings(index)...) want = append(want, record.Operator{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match}) if !slices.Equal(target.Operators, want) { - t.Fatalf("TextSearch answers %+v, want %+v", target.Operators, want) + t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } case len(target.Operators) != 0: // Every other type is storage-only or not produced. diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 34b398af5..448dfdbfc 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -6,8 +6,8 @@ // to check one. // // The spellings here are wire format, fixed by stack-encrypt's -// dynamic::record: the output keys "c", "eq", "match", "ore", "ope" and -// "passthrough", and the type names "bool", "int8", "int16", "int32", +// dynamic::record: the output keys "c", "eq", "match", "ore", "ope", +// "ore_block" and "passthrough", and the type names "bool", "int8", "int16", "int32", // "int64", "int128", "uint8", "uint16", "uint32", "uint64", "uint128", // "float32", "float64", "decimal", "date", "timestamp", "string" and // "bytes". A field's context is @@ -75,6 +75,7 @@ const ( Match Output = "match" Ore Output = "ore" Ope Output = "ope" + OreBlock Output = "ore_block" Passthrough Output = "passthrough" ) @@ -428,7 +429,7 @@ type Sealed = map[string]Outputs // Target is one EQL type as se_targets lists it. The entry's keys are wire // format, written by eql-bindings' serialiser and read by [ParseTargets]: // name, family, suffix, plaintext (a ValueKind name, or null), sql_domain, -// indexes (eq | match | ore | ope | json), query (or null), query_sql_domain +// indexes (eq | match | ore | ope | ore_block | json), query (or null), query_sql_domain // (or null), producible, reason (or null), and operators, which the engine // adds: the operators it answers for the type, each an object of operator, // fidelity (exact | probabilistic), arity (one | pair) and index. diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index eff71aa49..273a7c978 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -154,6 +154,11 @@ const ( IndexOre IndexName = "ore" IndexOpe IndexName = "ope" IndexJSON IndexName = "json" + // IndexOreBlock is block ORE (ore-rs's 6-bit schemes): the index of an + // EQL type's ORE term (`ob`), TextOrdOre's and IntegerOrdOre's. It is + // not a tag word: a field reaches it by naming such an EQL type + // (encrypt_into=), and indexNames leaves it out. + IndexOreBlock IndexName = "ore_block" ) // indexNames lists every index, in the order the generated type lists their @@ -172,6 +177,8 @@ func (n IndexName) GoName() string { return "Ore" case IndexOpe: return "Ope" + case IndexOreBlock: + return "OreBlock" case IndexJSON: return "JSON" } diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index d8b71bca7..50255cf33 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -277,8 +277,11 @@ var outputOfIndex = map[IndexName]record.Output{ IndexEquality: record.Equality, IndexMatch: record.Match, IndexOre: record.Ore, IndexOpe: record.Ope, } +// indexOfOutput also names block ORE, which only an EQL type carries: it has +// no entry in outputOfIndex, so no tag asks for it. var indexOfOutput = map[record.Output]IndexName{ record.Equality: IndexEquality, record.Match: IndexMatch, record.Ore: IndexOre, record.Ope: IndexOpe, + record.OreBlock: IndexOreBlock, } // wireKind is the record kind of a Go type: the same mapping the emitter diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 166b0bc6e..61be14f85 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -34,6 +34,9 @@ var eqlTypes = []struct { {stashgen.EQLType{Name: "TextEq", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality}, Query: "TextEqQuery", Producible: true}, ""}, {stashgen.EQLType{Name: "TextMatch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexMatch}, Query: "TextMatchQuery", Producible: true}, ""}, {stashgen.EQLType{Name: "TextSearch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality, stashgen.IndexOpe, stashgen.IndexMatch}, Query: "TextSearchQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextOrdOre", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality, stashgen.IndexOreBlock}, Query: "TextOrdOreQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextSearchOre", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality, stashgen.IndexOreBlock, stashgen.IndexMatch}, Query: "TextSearchOreQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "IntegerOrdOre", Plaintext: stashgen.KindInt, Indexes: []stashgen.IndexName{stashgen.IndexOreBlock}, Query: "IntegerOrdOreQuery", Producible: true}, "int32"}, {stashgen.EQLType{Name: "Boolean", Plaintext: stashgen.KindBool, Producible: true}, ""}, {stashgen.EQLType{Name: "Real", Plaintext: stashgen.KindFloat, Producible: true}, "float32"}, {stashgen.EQLType{Name: "Double", Plaintext: stashgen.KindFloat, Producible: true}, "float64"}, @@ -183,7 +186,7 @@ func (e Static) operators(ctx context.Context, d stashgen.Declaration) (map[stri // answeredBy mirrors OpSpec::answered_by in stack-encrypt: eq through an // equality index, matches through a match index, and the orderings through -// ORE, or OPE when there is no ORE, over an integer or text; each only +// ORE, else block ORE, else OPE, over an integer or text; each only // where the index applies to the type. func answeredBy(operator string, t stashgen.GoType, indexes []stashgen.IndexName) (stashgen.IndexName, bool) { declared := func(want stashgen.IndexName) bool { @@ -204,8 +207,10 @@ func answeredBy(operator string, t stashgen.GoType, indexes []stashgen.IndexName if !orderable { return "", false } - if declared(stashgen.IndexOre) { - return stashgen.IndexOre, true + for _, index := range []stashgen.IndexName{stashgen.IndexOre, stashgen.IndexOreBlock} { + if declared(index) { + return index, true + } } return stashgen.IndexOpe, declared(stashgen.IndexOpe) } @@ -253,7 +258,7 @@ func indexApplies(name stashgen.IndexName, t stashgen.GoType) error { return nil } return fmt.Errorf("match applies to a string, not to %s", t) - case stashgen.IndexOre, stashgen.IndexOpe: + case stashgen.IndexOre, stashgen.IndexOpe, stashgen.IndexOreBlock: switch t.Kind { case stashgen.KindString, stashgen.KindBytes, stashgen.KindFloat: return nil diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index d91f5de3e..027e45a02 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -93,7 +93,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with an interface", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAny any"), stashgen.Request{Type: "User"}, "Any", "is not a type JSON carries"}, {"an opaque struct with a struct JSON would truncate", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx hidden") + "\ntype hidden struct {\n\tA int\n\tb int\n}\n", stashgen.Request{Type: "User"}, "Tx", "JSON would drop"}, {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, - {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int32 `stash:\"age,encrypt_into=IntegerOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type IntegerOrdOre"}, + {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, {"the json index, not in the engine yet", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "cannot derive the json index yet"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index e85b3af88..a7012b2a1 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -582,8 +582,8 @@ dependencies = [ "log", "miette", "opaque-debug", - "orderable-bytes", - "ore-rs", + "orderable-bytes 0.1.1", + "ore-rs 0.8.3", "percent-encoding", "rand 0.8.6", "recipher 0.2.3", @@ -650,7 +650,7 @@ dependencies = [ "blake3", "chrono", "hex", - "orderable-bytes", + "orderable-bytes 0.1.1", "rust_decimal", "subtle", "thiserror 1.0.69", @@ -2667,6 +2667,12 @@ dependencies = [ "rust_decimal", ] +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + [[package]] name = "ore-rs" version = "0.8.3" @@ -2680,7 +2686,7 @@ dependencies = [ "hex", "lazy_static", "num", - "orderable-bytes", + "orderable-bytes 0.1.1", "rand 0.8.6", "rand_chacha 0.3.1", "rust_decimal", @@ -2689,6 +2695,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes 0.2.1", + "rand 0.8.6", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "outref" version = "0.5.2" @@ -4239,6 +4268,7 @@ dependencies = [ "base64ct", "cllw-ore 0.5.2-rc.1", "miette", + "ore-rs 0.9.0", "serde", "serde_json", "stack-encrypt-derive", diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index ab1637533..e1a655afe 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -118,6 +118,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `Into`), so it carries only the term its operator reads: `=` and `<` on two-letter text work, and a match query on it is refused. The by-name dispatch picks the projection for the index it is handed. +- **The block ORE types are producible** (`stack-encrypt` feature): + `IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre` and + `DoubleOrdOre` from `i32`, `i16`, `i64`, `f32` and `f64`, and `TextOrdOre` and `TextSearchOre` from + `String`. Their `ob` term is Stack Encrypt's block ORE term (ore-rs's + 6-bit fixed-width scheme for the numbers, its chained scheme for text): + one element, the hex of the whole ciphertext. `OreBlock256` implements + `EncryptFrom` for any orderable plaintext and never decrypts. This is not + EQL v3's `ob` (cipherstash-client's 8-bit blocks, up to six per text + value), and only EQL v4's comparator, `compare_ore_block_v2_term`, reads + it. Every term carries a random nonce, so a query's `ob` never equals the + stored one; the comparator orders them. A query carries the full + ciphertext. `TextSearchOre` follows `TextSearch`: its query carries only + the term its operator reads. Text whose fold is longer than 1023 bytes or + ends in NUL has no block ORE term and is refused. In the target table an + `ob` term is the index `ore_block`, not `ore`, which is CLLW ORE. - **`encryption::targets::query` takes the index that answers the query's operator** (`targets` feature), an `&IndexSpec` after the name, as Stack Encrypt's `TargetResolver::query` hands it over. An index the type does diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index b8e5dbf3f..1146823b4 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -112,13 +112,13 @@ plaintext, or, for a date, a timestamp or a decimal, one of | v4 type | Plaintext | | --- | --- | -| `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe` | `String` | -| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe` | `i16` | -| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe` | `i32` | -| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe` | `i64` | +| `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe`, `TextOrdOre`, `TextSearch`, `TextSearchOre` | `String` | +| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | `i16` | +| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | `i32` | +| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | `i64` | | `Boolean` | `bool` | -| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe` | `f32` | -| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe` | `f64` | +| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `f32` | +| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `f64` | | `Date`, `DateEq` | `v4::plaintext::Date`: an `i32` of days since 1970-01-01 | | `Timestamp`, `TimestampEq` | `v4::plaintext::Timestamp`: `i64` Unix seconds and `u32` nanoseconds, UTC, no leap seconds | | `Numeric`, `NumericEq` | `v4::plaintext::Decimal`: an `i128` mantissa (below 2^96) and a `u8` scale (at most 28) | @@ -175,9 +175,18 @@ through `TextSearchEqualityQuery` (`=`, `<>`), `TextSearchOrderQuery` (`<`, answers the operator. So `=` and `<` work on short text, a match query on it is refused, and EQL v4 raises on an operator whose term the query lacks (or holds as `null`, or as an empty filter). -The ORE types (`*OrdOre`, `TextSearchOre`) wait for block ORE, `Date`, -`Timestamp` and `Numeric` for a choice of their Rust plaintext types, and -`Json` for a decision on sealing a document as one leaf. +The ORE types (`*OrdOre`, `TextSearchOre`) store a block ORE term (`ob`): +one element, the hex of a whole ore-rs 6-bit block ORE ciphertext (the +fixed-width scheme for numbers, the chained scheme for text, padded to a +power-of-two bucket of at least 16 bytes). They order by the same key as the +`Ord` types, but EQL v4 compares them with its own comparator, +`eql_v4_internal.compare_ore_block_v2_term`, and `ORDER BY` reads +`eql_v4.ord_term_ore(col)`. Every term carries a random nonce, so a query's +`ob` never equals the stored bytes; the comparator orders them. Text whose +fold is longer than 1023 bytes or ends in NUL has no block ORE term and is +refused. `TextSearchOre` follows `TextSearch`, with `ob` in place of `op`. +`Date`, `Timestamp` and `Numeric` wait for a choice of their Rust plaintext +types, and `Json` for a decision on sealing a document as one leaf. Each ordered v4 family also has two range query types, one per order term: `IntegerRangeOreQuery` and `IntegerRangeOpeQuery`, and so on diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index a52bcd357..457ae3fb5 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -101,7 +101,7 @@ use base64::{engine::general_purpose::STANDARD, Engine as _}; #[allow(deprecated)] use stack_encrypt::sem::MatchTerm; use stack_encrypt::sem::{ - EqualityTerm, MatchConfig, MatchOptions, OpeTerm, Orderable, TermError, Tokenizer, + EqualityTerm, MatchConfig, MatchOptions, OpeTerm, Orderable, OreBlockTerm, TermError, Tokenizer, }; use stack_encrypt::target::transcode::{Reader, Transcode, Visitor}; use stack_encrypt::target::{self, AeadContext, CallerContext, IndexSpec}; @@ -112,7 +112,7 @@ use stack_encrypt::{ }; use crate::{ - v3::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw}, + v3::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256}, v4::terms::SearchBloomFilter, Identifier, }; @@ -495,6 +495,52 @@ impl DecryptField for OpeCllw { } } +/// Builds EQL v4's `ob` from the native block ORE term: one element, the +/// hex of the whole ore-rs ciphertext. +/// +/// The ciphertext is ore-rs's 6-bit block ORE of the value's ADR-0002 order +/// key (`OreBlockTerm`): the fixed-width scheme for integers and floats, the +/// chained one, padded to a power-of-two length bucket of at least 16 +/// bytes, for text. EQL v4 compares `ob` with the comparator for that +/// format, which takes exactly one element; EQL v3's `ob` holds 8-bit block +/// terms, up to six for text, that cipherstash-client writes. A query +/// carries the full ciphertext too: the comparator reads the right half of +/// the operand a btree index or `between` passes second. +/// +/// Text with no fold, text whose fold ends in NUL or is longer than 1023 +/// bytes has no block ORE term and is refused, stored or queried. +#[doc(hidden)] +pub struct OreBlockVisitor; +impl Visitor for OreBlockVisitor { + type Value = OreBlock256; + fn ore_block(self, term: OreBlockTerm) -> Result { + Ok(OreBlock256(vec![hex::encode(term.as_bytes())])) + } +} +impl Transcode for OreBlock256 { + type Visitor = OreBlockVisitor; + fn visitor() -> Self::Visitor { + OreBlockVisitor + } +} +impl EncryptFrom for OreBlock256 { + type Context = CallerContext; + fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> + where + S: 's, + { + target::ore_block().transcode() + } +} +impl Decryptable for OreBlock256 { + const DECRYPTABLE: bool = false; +} +impl DecryptField for OreBlock256 { + fn decryption_field(self, _: Ctx) -> Option> { + None + } +} + #[cfg(test)] mod tests { use stack_encrypt::{nonempty, Descriptor, Label}; diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 3447065d0..79912347a 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -731,10 +731,11 @@ mod tests { fn refusals_name_the_type_and_its_reason() { assert!(matches!(refuse("Nope"), TargetError::Unknown { name } if name == "Nope")); assert!(matches!( - refuse("TextOrdOre"), - TargetError::Unproducible { name: "TextOrdOre", reason } if reason.contains("block ORE") + refuse("DateOrdOre"), + TargetError::Unproducible { name: "DateOrdOre", reason } if reason.contains("no order term for a date, timestamp or decimal") )); assert!(producible("TextEq").is_ok()); + assert!(producible("TextOrdOre").is_ok()); } fn frame(value: T) -> ValueFrame { diff --git a/packages/eql/crates/eql-bindings/src/v4/bigint.rs b/packages/eql/crates/eql-bindings/src/v4/bigint.rs index 359f1fff1..740d6c88d 100644 --- a/packages/eql/crates/eql-bindings/src/v4/bigint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/bigint.rs @@ -113,10 +113,18 @@ impl DomainType for BigintEq { doc, doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = i64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct BigintOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for BigintEqQuery { doc, doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`i64`](i64). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = i64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct BigintOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/double.rs b/packages/eql/crates/eql-bindings/src/v4/double.rs index c4aa41357..f2e00f4d9 100644 --- a/packages/eql/crates/eql-bindings/src/v4/double.rs +++ b/packages/eql/crates/eql-bindings/src/v4/double.rs @@ -113,10 +113,18 @@ impl DomainType for DoubleEq { doc, doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = f64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DoubleOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for DoubleEqQuery { doc, doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`f64`](f64). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = f64)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DoubleOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/integer.rs b/packages/eql/crates/eql-bindings/src/v4/integer.rs index c0f661fe9..8fcb2c945 100644 --- a/packages/eql/crates/eql-bindings/src/v4/integer.rs +++ b/packages/eql/crates/eql-bindings/src/v4/integer.rs @@ -113,10 +113,18 @@ impl DomainType for IntegerEq { doc, doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = i32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct IntegerOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for IntegerEqQuery { doc, doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`i32`](i32). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = i32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct IntegerOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/real.rs b/packages/eql/crates/eql-bindings/src/v4/real.rs index 3def16c0e..945374401 100644 --- a/packages/eql/crates/eql-bindings/src/v4/real.rs +++ b/packages/eql/crates/eql-bindings/src/v4/real.rs @@ -113,10 +113,18 @@ impl DomainType for RealEq { doc, doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = f32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RealOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for RealEqQuery { doc, doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`f32`](f32). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = f32)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RealOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/smallint.rs b/packages/eql/crates/eql-bindings/src/v4/smallint.rs index a3f7eb236..503f7bf7b 100644 --- a/packages/eql/crates/eql-bindings/src/v4/smallint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/smallint.rs @@ -113,10 +113,18 @@ impl DomainType for SmallintEq { doc, doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = i16)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SmallintOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub ob: OreBlock256, @@ -304,10 +312,18 @@ impl DomainType for SmallintEqQuery { doc, doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`i16`](i16). Implements `EncryptFrom` to build order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = i16)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SmallintOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub ob: OreBlock256, } diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index 698748bfe..feddec846 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -46,13 +46,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("int32"), sql_domain: "public.eql_v4_integer_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("IntegerOrdOreQuery"), query_sql_domain: Some("eql_v4.query_integer_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "IntegerOrd", @@ -108,13 +106,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("int16"), sql_domain: "public.eql_v4_smallint_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("SmallintOrdOreQuery"), query_sql_domain: Some("eql_v4.query_smallint_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "SmallintOrd", @@ -170,13 +166,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("int64"), sql_domain: "public.eql_v4_bigint_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("BigintOrdOreQuery"), query_sql_domain: Some("eql_v4.query_bigint_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "BigintOrd", @@ -232,7 +226,7 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("date"), sql_domain: "public.eql_v4_date_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("DateOrdOreQuery"), query_sql_domain: Some("eql_v4.query_date_ord_ore"), producible: false, @@ -298,7 +292,7 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("timestamp"), sql_domain: "public.eql_v4_timestamp_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("TimestampOrdOreQuery"), query_sql_domain: Some("eql_v4.query_timestamp_ord_ore"), producible: false, @@ -364,7 +358,7 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("decimal"), sql_domain: "public.eql_v4_numeric_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("NumericOrdOreQuery"), query_sql_domain: Some("eql_v4.query_numeric_ord_ore"), producible: false, @@ -442,13 +436,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("string"), sql_domain: "public.eql_v4_text_ord_ore", - indexes: &["eq", "ore"], + indexes: &["eq", "ore_block"], query: Some("TextOrdOreQuery"), query_sql_domain: Some("eql_v4.query_text_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "TextOrd", @@ -480,13 +472,11 @@ pub const TARGETS: &[Target] = &[ suffix: "SearchOre", plaintext: Some("string"), sql_domain: "public.eql_v4_text_search_ore", - indexes: &["eq", "ore", "match"], + indexes: &["eq", "ore_block", "match"], query: Some("TextSearchOreQuery"), query_sql_domain: Some("eql_v4.query_text_search_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "TextSearch", @@ -542,13 +532,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("float32"), sql_domain: "public.eql_v4_real_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("RealOrdOreQuery"), query_sql_domain: Some("eql_v4.query_real_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "RealOrd", @@ -604,13 +592,11 @@ pub const TARGETS: &[Target] = &[ suffix: "OrdOre", plaintext: Some("float64"), sql_domain: "public.eql_v4_double_ord_ore", - indexes: &["ore"], + indexes: &["ore_block"], query: Some("DoubleOrdOreQuery"), query_sql_domain: Some("eql_v4.query_double_ord_ore"), - producible: false, - reason: Some( - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)", - ), + producible: true, + reason: None, }, Target { name: "DoubleOrd", @@ -679,6 +665,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "IntegerEq" => { run_target::("IntegerEq", keyset, column, plaintext) } + "IntegerOrdOre" => run_target::( + "IntegerOrdOre", + keyset, + column, + plaintext, + ), "IntegerOrd" => run_target::( "IntegerOrd", keyset, @@ -700,6 +692,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), + "SmallintOrdOre" => run_target::( + "SmallintOrdOre", + keyset, + column, + plaintext, + ), "SmallintOrd" => run_target::( "SmallintOrd", keyset, @@ -718,6 +716,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "BigintEq" => { run_target::("BigintEq", keyset, column, plaintext) } + "BigintOrdOre" => run_target::( + "BigintOrdOre", + keyset, + column, + plaintext, + ), "BigintOrd" => { run_target::("BigintOrd", keyset, column, plaintext) } @@ -757,6 +761,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "TextMatch" => { run_target::("TextMatch", keyset, column, plaintext) } + "TextOrdOre" => run_target::( + "TextOrdOre", + keyset, + column, + plaintext, + ), "TextOrd" => { run_target::("TextOrd", keyset, column, plaintext) } @@ -766,6 +776,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), + "TextSearchOre" => run_target::( + "TextSearchOre", + keyset, + column, + plaintext, + ), "TextSearch" => run_target::( "TextSearch", keyset, @@ -777,6 +793,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( } "Real" => run_target::("Real", keyset, column, plaintext), "RealEq" => run_target::("RealEq", keyset, column, plaintext), + "RealOrdOre" => { + run_target::("RealOrdOre", keyset, column, plaintext) + } "RealOrd" => { run_target::("RealOrd", keyset, column, plaintext) } @@ -789,6 +808,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "DoubleEq" => { run_target::("DoubleEq", keyset, column, plaintext) } + "DoubleOrdOre" => run_target::( + "DoubleOrdOre", + keyset, + column, + plaintext, + ), "DoubleOrd" => { run_target::("DoubleOrd", keyset, column, plaintext) } @@ -816,6 +841,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "IntegerEq" => { open_target::("IntegerEq", opener, column, stored) } + "IntegerOrdOre" => open_target::( + "IntegerOrdOre", + opener, + column, + stored, + ), "IntegerOrd" => { open_target::("IntegerOrd", opener, column, stored) } @@ -831,6 +862,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "SmallintEq" => { open_target::("SmallintEq", opener, column, stored) } + "SmallintOrdOre" => open_target::( + "SmallintOrdOre", + opener, + column, + stored, + ), "SmallintOrd" => open_target::( "SmallintOrd", opener, @@ -847,6 +884,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "BigintEq" => { open_target::("BigintEq", opener, column, stored) } + "BigintOrdOre" => open_target::( + "BigintOrdOre", + opener, + column, + stored, + ), "BigintOrd" => { open_target::("BigintOrd", opener, column, stored) } @@ -884,12 +927,21 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "TextMatch" => { open_target::("TextMatch", opener, column, stored) } + "TextOrdOre" => { + open_target::("TextOrdOre", opener, column, stored) + } "TextOrd" => { open_target::("TextOrd", opener, column, stored) } "TextOrdOpe" => { open_target::("TextOrdOpe", opener, column, stored) } + "TextSearchOre" => open_target::( + "TextSearchOre", + opener, + column, + stored, + ), "TextSearch" => { open_target::("TextSearch", opener, column, stored) } @@ -898,6 +950,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( } "Real" => open_target::("Real", opener, column, stored), "RealEq" => open_target::("RealEq", opener, column, stored), + "RealOrdOre" => { + open_target::("RealOrdOre", opener, column, stored) + } "RealOrd" => open_target::("RealOrd", opener, column, stored), "RealOrdOpe" => { open_target::("RealOrdOpe", opener, column, stored) @@ -906,6 +961,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "DoubleEq" => { open_target::("DoubleEq", opener, column, stored) } + "DoubleOrdOre" => open_target::( + "DoubleOrdOre", + opener, + column, + stored, + ), "DoubleOrd" => { open_target::("DoubleOrd", opener, column, stored) } @@ -937,6 +998,12 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "IntegerOrdOre" => run_target::( + "IntegerOrdOre", + keyset, + column, + plaintext, + ), "IntegerOrd" => run_target::( "IntegerOrd", keyset, @@ -955,6 +1022,12 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "SmallintOrdOre" => run_target::( + "SmallintOrdOre", + keyset, + column, + plaintext, + ), "SmallintOrd" => run_target::( "SmallintOrd", keyset, @@ -970,6 +1043,12 @@ pub(crate) fn query_named<'a, K: 'static>( "BigintEq" => run_target::( "BigintEq", keyset, column, plaintext, ), + "BigintOrdOre" => run_target::( + "BigintOrdOre", + keyset, + column, + plaintext, + ), "BigintOrd" => run_target::( "BigintOrd", keyset, @@ -1006,6 +1085,12 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "TextOrdOre" => run_target::( + "TextOrdOre", + keyset, + column, + plaintext, + ), "TextOrd" => { run_target::("TextOrd", keyset, column, plaintext) } @@ -1015,6 +1100,27 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "TextSearchOre" => match index.key() { + "eq" => run_projected::< + super::text::TextSearchOreEqualityQuery, + super::text::TextSearchOreQuery, + String, + K, + >("TextSearchOre", keyset, column, plaintext), + "ore_block" => run_projected::< + super::text::TextSearchOreOrderQuery, + super::text::TextSearchOreQuery, + String, + K, + >("TextSearchOre", keyset, column, plaintext), + "match" => run_projected::< + super::text::TextSearchOreMatchQuery, + super::text::TextSearchOreQuery, + String, + K, + >("TextSearchOre", keyset, column, plaintext), + _ => Err(no_index("TextSearchOre", index)), + }, "TextSearch" => match index.key() { "eq" => run_projected::< super::text::TextSearchEqualityQuery, @@ -1039,6 +1145,12 @@ pub(crate) fn query_named<'a, K: 'static>( "RealEq" => { run_target::("RealEq", keyset, column, plaintext) } + "RealOrdOre" => run_target::( + "RealOrdOre", + keyset, + column, + plaintext, + ), "RealOrd" => { run_target::("RealOrd", keyset, column, plaintext) } @@ -1051,6 +1163,12 @@ pub(crate) fn query_named<'a, K: 'static>( "DoubleEq" => run_target::( "DoubleEq", keyset, column, plaintext, ), + "DoubleOrdOre" => run_target::( + "DoubleOrdOre", + keyset, + column, + plaintext, + ), "DoubleOrd" => run_target::( "DoubleOrd", keyset, diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index 2de5a1458..af5544826 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -163,10 +163,18 @@ impl DomainType for TextMatch { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextOrdOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, @@ -308,10 +316,18 @@ impl DomainType for TextOrdOpe { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearchOre { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, @@ -502,10 +518,18 @@ impl DomainType for TextMatchQuery { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality and order query terms. Query operands do not decrypt. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextOrdOreQuery { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub hm: Hmac256, pub ob: OreBlock256, @@ -644,6 +668,8 @@ impl DomainType for TextOrdOpeQuery { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`String`](String), through one projection per index — [`TextSearchOreEqualityQuery`], [`TextSearchOreOrderQuery`], [`TextSearchOreMatchQuery`] — each `EncryptFrom` and `Into`: a query carries the term of the index that answers its operator. Query operands do not decrypt. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearchOreQuery { @@ -682,6 +708,81 @@ impl DomainType for TextSearchOreQuery { format!("{}{}.json", super::SCHEMA_ID_BASE, self.domain()) } } +/// The `eql_v4.query_text_search_ore` query operand for `=` `<>`: the `hm` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchOreQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOreEqualityQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub hm: Hmac256, +} +impl From for TextSearchOreQuery { + fn from(query: TextSearchOreEqualityQuery) -> Self { + TextSearchOreQuery { + v: query.v, + i: query.i, + hm: Some(query.hm), + ob: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search_ore` query operand for `<` `<=` `>` `>=`: the `ob` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchOreQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOreOrderQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub ob: OreBlock256, +} +impl From for TextSearchOreQuery { + fn from(query: TextSearchOreOrderQuery) -> Self { + TextSearchOreQuery { + v: query.v, + i: query.i, + ob: Some(query.ob), + hm: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search_ore` query operand for `@@`: the `bf` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchOreQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOreMatchQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub bf: BloomFilter, +} +impl From for TextSearchOreQuery { + fn from(query: TextSearchOreMatchQuery) -> Self { + TextSearchOreQuery { + v: query.v, + i: query.i, + bf: Some(query.bf), + hm: None, + ob: None, + } + } +} /// `eql_v4.query_text_search` — search domain query operand. /// /// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i`, and at least one of `hm` `op` `bf`: a query carries the term its operator reads. diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 35a1fedc3..cb5011b29 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -574,22 +574,29 @@ fn render_query_projections( /// The `(family, domain)` pairs whose generated v4 structs carry the opt-in /// `stack-encrypt` derives: the storage and equality domains of every family -/// whose plaintext eql-bindings names (`plaintext_of`), the OPE-ordered -/// domains (`ord`, `ord_ope`) of those whose order terms the engine derives, -/// and text match and search. Every other stored domain has a reason in -/// [`encryption_gap`], and a test holds the two together so a new domain +/// whose plaintext eql-bindings names (`plaintext_of`), the ordered domains +/// (`ord` and `ord_ope` with an OPE term, `ord_ore` with a block ORE term) of +/// those whose order terms the engine derives, and text match and search +/// (`search` and `search_ore`). Every other stored domain has a reason +/// in [`encryption_gap`], and a test holds the two together so a new domain /// cannot arrive without either a derive or a reason. +/// +/// Listed in catalog order, which is the order the generated target table +/// lists them in. pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ ("integer", ""), ("integer", "eq"), + ("integer", "ord_ore"), ("integer", "ord"), ("integer", "ord_ope"), ("smallint", ""), ("smallint", "eq"), + ("smallint", "ord_ore"), ("smallint", "ord"), ("smallint", "ord_ope"), ("bigint", ""), ("bigint", "eq"), + ("bigint", "ord_ore"), ("bigint", "ord"), ("bigint", "ord_ope"), ("date", ""), @@ -601,16 +608,20 @@ pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ ("text", ""), ("text", "eq"), ("text", "match"), + ("text", "ord_ore"), ("text", "ord"), ("text", "ord_ope"), + ("text", "search_ore"), ("text", "search"), ("boolean", ""), ("real", ""), ("real", "eq"), + ("real", "ord_ore"), ("real", "ord"), ("real", "ord_ope"), ("double", ""), ("double", "eq"), + ("double", "ord_ore"), ("double", "ord"), ("double", "ord_ope"), ]; @@ -673,17 +684,12 @@ pub fn encryption_gap( } else if ORDER_PENDING_FAMILIES.contains(&family.name) && (domain.terms.contains(&Term::Ope) || domain.terms.contains(&Term::Ore)) { - // Before the ORE branch: the engine derives no order term of either - // scheme for these kinds, so block ORE is not the first thing a - // `*OrdOre` type of these families waits for. + // The engine derives no order term of either scheme for these kinds. "the engine derives no order term for a date, timestamp or decimal yet: how they are \ ordered is not decided (#1236)" - } else if domain.terms.contains(&Term::Ore) { - "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)" } else { // Reached by any unlisted domain of a family whose plaintext is - // specified and whose terms are not ORE (an equality or OPE-ordered - // domain of a family that gains a plaintext, say): nothing in the + // specified (a domain added to such a family, say): nothing in the // catalog today. "this domain derives the way its family's other domains do; it is not listed in \ ENCRYPTION_DOMAINS yet" @@ -713,7 +719,7 @@ fn encryption_attrs( let terms = match ( domain.terms.contains(&Term::Bloom), domain.terms.contains(&Term::Hm), - domain.terms.contains(&Term::Ope), + domain.terms.contains(&Term::Ope) || domain.terms.contains(&Term::Ore), ) { (true, _, _) => "match", (false, true, true) => "equality and order", @@ -2084,10 +2090,8 @@ mod tests { "no Rust plaintext type for this family" } else if ORDER_PENDING_FAMILIES.contains(&family.name) { "no order term for a date, timestamp or decimal" - } else if domain.terms.contains(&Term::Ore) { - "block ORE terms" } else { - panic!("{full}: every specified domain without ORE is derived: {reason}") + panic!("{full}: every domain of a specified family is derived: {reason}") }; assert!(reason.contains(expected), "{full}: {reason}"); } diff --git a/packages/eql/crates/eql-codegen/src/consts.rs b/packages/eql/crates/eql-codegen/src/consts.rs index dbbae25a1..6cca18dff 100644 --- a/packages/eql/crates/eql-codegen/src/consts.rs +++ b/packages/eql/crates/eql-codegen/src/consts.rs @@ -48,6 +48,23 @@ pub(crate) const PUBLIC_TYPNAME_PREFIX: &str = "{{prefix}}_"; /// The payload version every domain CHECK compares `VALUE->>'v'` against. pub(crate) const EQL_VERSION: &str = "{{eql_version}}"; +/// The block ORE term type's name, which differs between the versions: +/// `ore_block_256` in `eql_v3`, `ore_block_v2` in `eql_v4` +/// ([`crate::render`]). [`eql_domains::Term::ctor`] stays the `eql_v3` name, +/// which the Rust bindings and the catalog dump report; the generated SQL +/// writes this placeholder in its place ([`sql_ctor`]). +pub(crate) const ORE_BLOCK: &str = "{{ore_block}}"; + +/// The constructor name a term's generated SQL calls: [`ORE_BLOCK`] for the +/// block ORE term, whose name follows the version, and +/// [`eql_domains::Term::ctor`] for the rest. +pub(crate) fn sql_ctor(term: eql_domains::Term) -> &'static str { + match term { + eql_domains::Term::Ore => ORE_BLOCK, + other => other.ctor(), + } +} + /// Always-present payload keys checked for presence in every domain CHECK. /// Term-specific keys are appended after these by `context::domain_block`. /// Defined in the catalog (`eql_domains::ENVELOPE_KEYS`) so the CHECKs and diff --git a/packages/eql/crates/eql-codegen/src/context.rs b/packages/eql/crates/eql-codegen/src/context.rs index 392507933..67c820214 100644 --- a/packages/eql/crates/eql-codegen/src/context.rs +++ b/packages/eql/crates/eql-codegen/src/context.rs @@ -73,6 +73,7 @@ pub fn environment() -> minijinja::Environment<'static> { env.add_global("schema", SCHEMA); env.add_global("internal_schema", INTERNAL_SCHEMA); env.add_global("eql_version", EQL_VERSION); + env.add_global("ore_block", ORE_BLOCK); // The renderer's version-block directives, for SQL that differs between // `eql_v3` and `eql_v4` (`crate::render`, "Version blocks"). env.add_global("v3_only", block_open(EqlVersion::V3)); @@ -335,7 +336,7 @@ pub struct FunctionsContext { /// Build the inlinable index-extractor entry for a domain term. /// /// The `RETURNS` type name equals the constructor name (`hmac_256`, -/// `ore_block_256`); qualify it with `INTERNAL_SCHEMA` — the same schema as +/// `{{ore_block}}`, [`sql_ctor`]); qualify it with `INTERNAL_SCHEMA` — the same schema as /// the body's constructor call — so the declared return type and the call /// stay in lockstep. `Term::returns()` is intentionally not used. pub fn extractor_entry(term: Term) -> FnEntry { @@ -351,9 +352,9 @@ pub fn query_extractor_entry(domain: &Domain, term: Term, query_dom: &str) -> Fn fn extractor_entry_requiring(term: Term, required: Option) -> FnEntry { FnEntry::Extractor { - ret: format!("{INTERNAL_SCHEMA}.{}", term.ctor()), + ret: format!("{INTERNAL_SCHEMA}.{}", sql_ctor(term)), extractor: term.extractor().to_string(), - ctor: term.ctor().to_string(), + ctor: sql_ctor(term).to_string(), required, } } diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 1f4d41634..4ef57020e 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -241,10 +241,13 @@ mod tests { // so no type is declared twice. assert_eq!(out.matches("\ntype JSON []byte\n").count(), 1); assert_eq!(out.matches("\ntype SteVecQuery []byte\n").count(), 1); - // TextEq is producible and says so; TextOrdOre is not and says why. + // TextEq is producible and says so; DateOrdOre is not and says why. assert!(out.contains("// TextEq is the EQL type stored in public.eql_v4_text_eq")); assert!(out.contains("Producible: true, Reason: \"\"")); - assert!(out.contains("Name: \"TextOrdOre\"") && out.contains("block ORE terms")); + assert!( + out.contains("Name: \"DateOrdOre\"") + && out.contains("no order term for a date, timestamp or decimal") + ); // Every table row is one line: the shape gofmt leaves alone. for line in out.lines().filter(|l| l.starts_with("\t{Name: ")) { assert!(line.ends_with("},"), "{line}"); @@ -269,7 +272,7 @@ mod tests { } assert_eq!(count, 18, "nine ordered families, ORE and OPE each"); assert!(out.contains( - "// IntegerRangeOreQuery is a range query value over the ore index of the integer family: \ + "// IntegerRangeOreQuery is a range query value over the ore_block index of the integer family: \ both bounds of one BETWEEN,\n// included, as the one operand eql_v4.query_integer_range_ore \ takes in eql_v4.between. It ranges over IntegerOrdOre.\n" )); diff --git a/packages/eql/crates/eql-codegen/src/range.rs b/packages/eql/crates/eql-codegen/src/range.rs index 8c31f5d13..a231c7c9e 100644 --- a/packages/eql/crates/eql-codegen/src/range.rs +++ b/packages/eql/crates/eql-codegen/src/range.rs @@ -267,7 +267,7 @@ pub fn render_range_functions_file(spec: &DomainFamily) -> Option { }; for &term in &terms { let range = spec.range_query_name(term); - let returns = format!("{INTERNAL_SCHEMA}.{}", term.ctor()); + let returns = format!("{INTERNAL_SCHEMA}.{}", crate::consts::sql_ctor(term)); for (side, function_name, bound) in [ ("low", "lo", RANGE_BOUND_KEYS[0]), ("high", "hi", RANGE_BOUND_KEYS[1]), @@ -280,7 +280,7 @@ pub fn render_range_functions_file(spec: &DomainFamily) -> Option { returns: returns.clone(), body: format!( "{INTERNAL_SCHEMA}.{}(jsonb_build_object('{}', r::jsonb -> '{}'))", - term.ctor(), + crate::consts::sql_ctor(term), sql_str(term.json_key()), sql_str(bound) ), @@ -456,9 +456,9 @@ mod tests { for (side, bound) in [("lo", "lo"), ("hi", "hi")] { assert!(v4.contains(&format!( "CREATE FUNCTION eql_v4_internal.{side}(r eql_v4.query_integer_range_ore)\n\ - RETURNS eql_v4_internal.ore_block_256\n\ + RETURNS eql_v4_internal.ore_block_v2\n\ LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n\ - AS $$ SELECT eql_v4_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> '{bound}')) $$;" + AS $$ SELECT eql_v4_internal.ore_block_v2(jsonb_build_object('ob', r::jsonb -> '{bound}')) $$;" )), "{side} ORE"); assert!(v4.contains(&format!( "CREATE FUNCTION eql_v4_internal.{side}(r eql_v4.query_integer_range_ope)\n\ @@ -582,8 +582,9 @@ mod tests { )), "{v4}" ); - assert!(v4 - .contains("AND c.opcintype = 'eql_v4_internal.ore_block_256'::pg_catalog.regtype")); + assert!( + v4.contains("AND c.opcintype = 'eql_v4_internal.ore_block_v2'::pg_catalog.regtype") + ); } } diff --git a/packages/eql/crates/eql-codegen/src/render.rs b/packages/eql/crates/eql-codegen/src/render.rs index 2441ce360..c37247164 100644 --- a/packages/eql/crates/eql-codegen/src/render.rs +++ b/packages/eql/crates/eql-codegen/src/render.rs @@ -10,10 +10,19 @@ //! |-------------------|----------|----------| //! | `{{prefix}}` | `eql_v3` | `eql_v4` | //! | `{{eql_version}}` | `3` | `4` | +//! | `{{ore_block}}` | `ore_block_256` | `ore_block_v2` | //! //! `{{prefix}}` is both the schema (`{{prefix}}.eq_term`) and the start of every //! other name (`{{prefix}}_internal`, `public.{{prefix}}_text_eq`). //! +//! `{{ore_block}}` names the block ORE term type and everything built on it +//! (`{{prefix}}_internal.{{ore_block}}_term`, `compare_{{ore_block}}_terms`). +//! The two versions compare different ciphertexts under the one `ob` key: +//! `eql_v3` cipherstash-client's 8-bit block ORE, whose blocks are 256-way, +//! and `eql_v4` ore-rs's 6-bit block ORE, whose wire format is ore-rs's +//! version 2. Each version's name says which it compares, so `eql_v4` does +//! not reuse the name of another format. +//! //! Rendering is strict in both directions. An unknown or unterminated //! placeholder is an error, and so is a literal `eql_v3` or `eql_v4` in the //! source: a literal name would render the same in both outputs, so the `eql_v4` @@ -62,6 +71,15 @@ pub const EQL_VERSION: &str = "{{eql_version}}"; /// to every version. The number is the payload version ([`EqlVersion::number`]), /// not the schema name, because the source may not name a schema literally. pub const ONLY_DIRECTIVE: &str = "-- ONLY EQL VERSION:"; +/// The block ORE name placeholder. See [the module docs](self). +pub const ORE_BLOCK: &str = "{{ore_block}}"; + +/// The `eql_v3` block ORE name, which only [`ORE_BLOCK`] may write: a literal +/// would name the 8-bit format in the `eql_v4` bundle too. The source +/// directory of that name (`src/v3/sem/ore_block_256/`) is a path, not a +/// name, and is allowed. +const V3_ORE_BLOCK: &str = "ore_block_256"; + /// The directive line that opens a block kept only in `version`'s output: /// `-- {{#v4}}`. See [the module docs](self#version-blocks). pub fn block_open(version: EqlVersion) -> String { @@ -132,6 +150,15 @@ impl EqlVersion { EqlVersion::V4 => "eql_v4", } } + + /// The block ORE name: `ore_block_256` (cipherstash-client's 8-bit + /// format) or `ore_block_v2` (ore-rs's 6-bit format, wire version 2). + pub fn ore_block(self) -> &'static str { + match self { + EqlVersion::V3 => V3_ORE_BLOCK, + EqlVersion::V4 => "ore_block_v2", + } + } } impl std::str::FromStr for EqlVersion { @@ -164,6 +191,8 @@ pub enum RenderError { only: EqlVersion, version: EqlVersion, }, + /// A literal `ore_block_256` where `{{ore_block}}` belongs. + LiteralOreBlock { line: usize }, /// A version block opened inside another; blocks do not nest. NestedBlock { line: usize }, /// A block close with no open block, or naming another version than the @@ -178,7 +207,8 @@ impl fmt::Display for RenderError { match self { RenderError::UnknownPlaceholder { line, name } => write!( f, - "line {line}: unknown placeholder {{{{{name}}}}} (expected {PREFIX} or {EQL_VERSION})" + "line {line}: unknown placeholder {{{{{name}}}}} (expected {PREFIX}, {EQL_VERSION} \ + or {ORE_BLOCK})" ), RenderError::Unterminated { line } => { write!(f, "line {line}: `{{{{` with no closing `}}}}`") @@ -207,6 +237,11 @@ impl fmt::Display for RenderError { only.prefix(), version.prefix() ), + RenderError::LiteralOreBlock { line } => write!( + f, + "line {line}: literal `{V3_ORE_BLOCK}`; write {ORE_BLOCK} in its place, in \ + comments too, so the name follows the EQL version's block ORE format" + ), RenderError::NestedBlock { line } => { write!(f, "line {line}: a version block opened inside another") } @@ -215,7 +250,10 @@ impl fmt::Display for RenderError { "line {line}: a version block close with no open block of that version" ), RenderError::UnclosedBlock { line } => { - write!(f, "line {line}: a version block opened here is never closed") + write!( + f, + "line {line}: a version block opened here is never closed" + ) } } } @@ -324,6 +362,9 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result }); } } + if contains_ore_block_name(line) { + return Err(RenderError::LiteralOreBlock { line: lineno }); + } match (directive(line), block) { (Some(Directive::Open(_)), Some(_)) => { return Err(RenderError::NestedBlock { line: lineno }) @@ -352,6 +393,7 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result match &after[..close] { "prefix" => out.push_str(version.prefix()), "eql_version" => out.push_str(version.number()), + "ore_block" => out.push_str(version.ore_block()), other => { return Err(RenderError::UnknownPlaceholder { line: lineno, @@ -381,6 +423,14 @@ fn contains_name(line: &str, prefix: &str) -> bool { }) } +/// Whether `line` names `ore_block_256` other than as the source directory +/// `sem/ore_block_256/`. +fn contains_ore_block_name(line: &str) -> bool { + line.match_indices(V3_ORE_BLOCK).any(|(at, _)| { + !(line[..at].ends_with("sem/") && line[at + V3_ORE_BLOCK.len()..].starts_with('/')) + }) +} + /// Why a file could not be rendered. #[derive(Debug)] pub enum RenderFileError { @@ -529,6 +579,35 @@ mod tests { ); } + /// The block ORE name follows the version, and a literal of the + /// `eql_v3` name is refused unless it is the source directory's path. + #[test] + fn the_block_ore_name_follows_the_version() { + let src = "CREATE TYPE {{prefix}}_internal.{{ore_block}}_term AS (bytes bytea);\n\ + -- REQUIRE: src/v3/sem/ore_block_256/types.sql\n"; + assert_eq!( + render_str(src, EqlVersion::V3).unwrap(), + "CREATE TYPE eql_v3_internal.ore_block_256_term AS (bytes bytea);\n\ + -- REQUIRE: src/v3/sem/ore_block_256/types.sql\n" + ); + assert_eq!( + render_str(src, EqlVersion::V4).unwrap(), + "CREATE TYPE eql_v4_internal.ore_block_v2_term AS (bytes bytea);\n\ + -- REQUIRE: src/v3/sem/ore_block_256/types.sql\n" + ); + for literal in [ + "SELECT {{prefix}}_internal.ore_block_256(x);\n", + "-- compares ore_block_256 terms\n", + "-- the directory sem/ore_block_256 alone\n", + ] { + assert_eq!( + render_str(literal, EqlVersion::V4), + Err(RenderError::LiteralOreBlock { line: 1 }), + "{literal:?}" + ); + } + } + #[test] fn rejects_unterminated_placeholder() { assert_eq!( diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index 7b56b91f8..6019c73b1 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -31,9 +31,8 @@ //! //! The reasons a type is not producible: eql-bindings names no Rust //! plaintext type for the family; the engine derives no order term for a -//! date, timestamp or decimal; EQL's ORE term is block ORE where the -//! engine derives CLLW ORE; and the JSON index is a new engine operation. -//! Each reason string names the issue that tracks it. +//! date, timestamp or decimal; and the JSON index is a new engine +//! operation. Each reason string names the issue that tracks it. //! //! The query dispatch takes the index that answers the query's operator. A //! twin whose terms are optional in EQL v4 (`TextSearchQuery`) is built @@ -54,15 +53,19 @@ use crate::render::EqlVersion; const TARGET_VERSION: EqlVersion = EqlVersion::V4; /// The `IndexSpec::key()` a catalog term rides under in a plan — the -/// `"eq"` / `"match"` / `"ore"` / `"ope"` strings of stack-encrypt's data -/// grammar — so a Go generator can match a target's indexes against the +/// `"eq"` / `"match"` / `"ore_block"` / `"ope"` strings of stack-encrypt's +/// data grammar — so a Go generator can match a target's indexes against the /// indexes a plan field may ask for by the same names. Exhaustive on /// purpose: a new catalog term must say which engine index it is. +/// +/// EQL's `ob` term is block ORE, the engine's `ore_block` index (ore-rs's +/// 6-bit schemes), not its `ore` index, which is CLLW ORE and has no EQL +/// domain. pub fn index_key(term: Term) -> &'static str { match term { Term::Hm => "eq", Term::Bloom => "match", - Term::Ore => "ore", + Term::Ore => "ore_block", Term::Ope => "ope", } } @@ -104,11 +107,14 @@ impl Answer { /// Why the typed query chain has no answer yet for a type carrying these /// indexes, by [`index_key`] name, or `None` when it answers: a query twin /// is assembled from the one term its type's index derived, so a twin that -/// needs an order term (`"ore"` or `"ope"`), or more than one term, is not +/// needs an order term (`"ore"`, `"ore_block"` or `"ope"`), or more than one term, is not /// built by it. The reason is the generator's, for its tests and docs; the /// generated file simply leaves the type out. pub(crate) fn typed_gap(indexes: &[&str]) -> Option<&'static str> { - if indexes.iter().any(|key| matches!(*key, "ore" | "ope")) { + if indexes + .iter() + .any(|key| matches!(*key, "ore" | "ore_block" | "ope")) + { Some("its twin carries an order term, which the typed query chain does not derive yet") } else if indexes.len() > 1 { Some("its twin carries more than one term, which the typed query chain does not assemble yet") @@ -711,15 +717,18 @@ mod tests { target_gap(family, family.domain_by_name(name).expect("domain")) }; let text = eql_domains::TEXT; - for producible in ["", "eq", "match", "ord", "ord_ope", "search"] { + for producible in [ + "", + "eq", + "match", + "ord", + "ord_ope", + "ord_ore", + "search", + "search_ore", + ] { assert!(gap(&text, producible).is_none(), "text {producible:?}"); } - for block_ore in ["ord_ore", "search_ore"] { - assert!( - gap(&text, block_ore).unwrap().contains("block ORE"), - "{block_ore}" - ); - } assert!(gap(&text, "match").is_none(), "TextMatch is producible"); let family = |name: &str| { eql_domains::CATALOG @@ -730,20 +739,16 @@ mod tests { for family_name in ["integer", "smallint", "bigint", "real", "double"] { let family = family(family_name); assert!(gap(family, "").is_none(), "{family_name}"); - for producible in ["eq", "ord", "ord_ope"] { + for producible in ["eq", "ord", "ord_ope", "ord_ore"] { assert!( gap(family, producible).is_none(), "{family_name} {producible}" ); } - assert!( - gap(family, "ord_ore").unwrap().contains("block ORE"), - "{family_name}" - ); } // A date, a timestamp and a decimal are stored and compared for - // equality; every ordered domain of theirs waits for an order term, - // block ORE included, which waits for that first. + // equality; every ordered domain of theirs, block ORE included, + // waits for an order term. for family_name in ["date", "timestamp", "numeric"] { let family = family(family_name); assert!(gap(family, "").is_none(), "{family_name}"); @@ -766,7 +771,7 @@ mod tests { // a plan field spells an index by exactly these. assert_eq!(index_key(Term::Hm), "eq"); assert_eq!(index_key(Term::Bloom), "match"); - assert_eq!(index_key(Term::Ore), "ore"); + assert_eq!(index_key(Term::Ore), "ore_block"); assert_eq!(index_key(Term::Ope), "ope"); } diff --git a/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 b/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 index 5c349bff7..e8903b167 100644 --- a/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/ore_fallback.sql.j2 @@ -8,7 +8,7 @@ --! --! Runs after the DO block in src/v3/sem/ore_block_256/operator_class.sql, --! which ATTEMPTS to create the default btree operator class for ---! {{ internal_schema }}.ore_block_256 and skips it on insufficient_privilege +--! {{ internal_schema }}.{{ ore_block }} and skips it on insufficient_privilege --! (CREATE OPERATOR CLASS requires superuser; managed platforms — cloud --! Supabase and most hosted Postgres — run the installer as a non-superuser --! role). When the class was created, this file is a no-op. @@ -45,7 +45,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{ internal_schema }}.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{ internal_schema }}.{{ ore_block }}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 b/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 index a0fa8a0ac..1754139b6 100644 --- a/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/range_functions.sql.j2 @@ -41,7 +41,7 @@ AS $$ SELECT {{ a.body }} $$; --! stop inlining. Inlined, the planner sees two comparisons on --! `{{ schema }}.{{ b.extractor }}(a)`, which a btree index on that expression --! serves as one range scan.{% if b.ore %} For this ORE range that index needs the ---! default btree operator class on {{ internal_schema }}.ore_block_256, which the +--! default btree operator class on {{ internal_schema }}.{{ ore_block }}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column ({{ b.ope_columns }}) diff --git a/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 b/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 index 82259afc1..4a4b87045 100644 --- a/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/range_types.sql.j2 @@ -59,7 +59,7 @@ END $$; {%- if ore_ranges %} --- Without the default btree operator class on {{ internal_schema }}.ore_block_256 +-- Without the default btree operator class on {{ internal_schema }}.{{ ore_block }} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{ internal_schema }}.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -72,7 +72,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{ internal_schema }}.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{ internal_schema }}.{{ ore_block }}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/docs/upgrading/v3.1.md b/packages/eql/docs/upgrading/v3.1.md new file mode 100644 index 000000000..9c8eb257a --- /dev/null +++ b/packages/eql/docs/upgrading/v3.1.md @@ -0,0 +1,190 @@ +# Upgrading to EQL 3.1 + +`3.1.0` is a minor release. It changes two things in the `eql_v4` surface that +a caller can meet without changing any SQL: which terms a text search query +must carry, and which block ORE ciphertext the `_ord_ore` and +`text_search_ore` domains hold. **The `eql_v3` bundle is unchanged, byte for +byte.** If you install only `eql_v3`, nothing here applies to you. + +Both changes land through Stack Encrypt as well as through the SQL: with +`3.1.0`, Stack Encrypt produces `eql_v4_text_search` and every block ORE +type (`_ord_ore` and `text_search_ore`), so a value written by it carries the +terms this release's `eql_v4` expects. The notes below are for anyone who +builds `eql_v4` payloads by hand, or who stored `eql_v4` block ORE terms +before this release. + +## TL;DR + +1. An `eql_v4` text search query carries only the term its operator reads, + and an operator reading a term the query does not carry **raises** + (`invalid_parameter_value`) instead of matching nothing or everything — + [U-001](#u-001-a-text-search-query-carries-only-the-terms-its-operator-reads). +2. The `eql_v4` block ORE term is ore-rs's 6-bit scheme, compared by a new + comparator named `ore_block_v2`. It **refuses** the 8-bit terms the + previous `eql_v4` bundle compared; rows and queries holding them must be + written again through Stack Encrypt — + [U-002](#u-002-the-block-ore-term-is-ore-rss-6-bit-scheme). + +## Compatibility + +| Surface | Status | +| --- | --- | +| `eql_v3`: every domain, operator, extractor and term | **Unchanged.** The v3 bundle is byte for byte what 3.0.6 shipped; its `ob` is still the 8-bit block ORE under `ore_block_256`. | +| `eql_v4.query_text_search`, `eql_v4.query_text_search_ore` | **Relaxed CHECK.** At least one of the terms (`hm`; `op` or `ob`; `bf`) is required, where all three were. See U-001. | +| `=`, `<>`, `<`, `<=`, `>`, `>=`, `@@` on `public.eql_v4_text_search` / `public.eql_v4_text_search_ore` | **Tightened error semantics.** A query without the term the operator reads raises `invalid_parameter_value` wherever the operand is read. See U-001. | +| Stored `public.eql_v4_text_search` / `public.eql_v4_text_search_ore` values | **Unchanged.** A stored value still carries every term. Text too short for a match term stores the empty filter, `"bf": []`, which `=` and `<` find and no `@@` query does. | +| `public.eql_v4_*_ord_ore`, `public.eql_v4_text_search_ore`, `eql_v4.query_*_ord_ore`, `eql_v4.query_*_range_ore` | **Term format changed.** The `ob` term is one ore-rs 6-bit block ORE ciphertext. The previous 8-bit layout is refused, not misread. Re-encryption of those columns is required where such rows exist. See U-002. | +| `eql_v4_internal.ore_block_256`, `compare_ore_block_256_term` | **Renamed** in the v4 bundle to `eql_v4_internal.ore_block_v2` and `compare_ore_block_v2_term`, with a new body. Only the installer and the operator class name them. | +| `eql_v4` OPE (`op`) terms, equality (`hm`) terms, ciphertexts (`c`) | **Unchanged** by this release. | +| Postgres 14, 15, 16, 17 | Supported. | +| Supabase / managed Postgres (no superuser) | As before: the block ORE domains need the operator class the installer creates, so a non-superuser install disables them ([U-003 in the 3.0 guide](./v3.0.md#u-003-non-superuser-installs-disable-the-ore-backed-domains)). U-002 only concerns installs where they are enabled. | + +## Upgrade notes + +### U-001: A text search query carries only the terms its operator reads + +**What changed.** `eql_v4.query_text_search` and +`eql_v4.query_text_search_ore` required every term of the type: `hm` for +`=`, `op` (or `ob`) for the orderings and `bf` for `@@`. They now require at +least one. Each operator reads one term, and the extractor it reads through +raises when the query does not carry it, carries it as JSON `null`, or +carries an empty array: + +```text +ERROR: query_text_search value has no "hm" term, the term = <> compare +SQLSTATE: 22023 (invalid_parameter_value) +``` + +The message names the domain, the term and the operators that read it, +never the operand. The raise happens wherever the operand is read, so a +statement whose predicate is never reached (a generic plan over an empty +table, say) returns no row and no error. + +**Why.** Text shorter than one 3-gram has no match term, so with every term +required a search column could never be asked `name = $1` or `name < $1` for +such text, although both have an answer. And a term that is present but +unreadable must not become a silent miss: before, a missing term read as +`NULL`, which compares equal to nothing and matches no row. + +**Who is affected.** A caller that builds query operands by hand, or that +checks them against the domain CHECK. Stack Encrypt builds the query for the +operator's index and carries that term alone; nothing changes for its +callers. Stored values are untouched: a stored search value still carries +every term, and a row stored before this release is found exactly as before. + +**What to do.** Nothing, for stored data. If you build a text search query +yourself, carry the term the operator reads; carrying the others as well is +still valid. Do not pass `null` or `[]` for a term you lack: omit it, and ask +only through operators whose term you carry. If you handled "no result" from +a query with a missing term, expect an error now. + +**Verification.** + +```sql +-- A query carrying one term is a valid operand (was refused by the CHECK): +SELECT '{"v": 4, "i": {"t": "users", "c": "name"}, "hm": "00"}' + ::jsonb::eql_v4.query_text_search; + +-- A query carrying no term is still refused by the CHECK (23514): +SELECT '{"v": 4, "i": {"t": "users", "c": "name"}}' + ::jsonb::eql_v4.query_text_search; +``` + +Then, on a table with at least one row, `WHERE name < $1` with the `hm`-only +operand above must fail with SQLSTATE `22023` and the message naming `op`. + +### U-002: The block ORE term is ore-rs's 6-bit scheme + +**What changed.** The `ob` term of every `eql_v4` block ORE domain is now +one ciphertext of ore-rs's 6-bit block ORE: the fixed-width scheme for +numbers, the chained scheme for text, which pads the text to a power-of-two +length bucket of at least 16 bytes. The v4 bundle names its block ORE objects +`ore_block_v2` (`eql_v4_internal.ore_block_v2_term`, +`eql_v4_internal.compare_ore_block_v2_term`) and compares them with a new +comparator that checks each term's shape before comparing, then runs the +same number of statements whatever the terms hold. It raises on: + +- a term that is not a 6-bit block ORE ciphertext, the previous 8-bit layout + included: `Malformed ORE block term: N bytes`; +- two terms of different schemes (fixed-width against chained): `ORE block + terms of different schemes`; +- two chained terms padded under different policies: `ORE block terms padded + under different policies`. + +The `eql_v3` bundle keeps the 8-bit scheme under `ore_block_256`, unchanged. + +**Why.** The 6-bit scheme is what Stack Encrypt derives, and with it Stack +Encrypt produces the block ORE types (`IntegerOrdOre`, `SmallintOrdOre`, +`BigintOrdOre`, `RealOrdOre`, `DoubleOrdOre`, `TextOrdOre`, +`TextSearchOre`) for the first time. One comparator per scheme, with the +scheme in the name, keeps a v3 term and a v4 term from ever being compared +by the other's rules. + +**Who is affected.** No Stack Encrypt release wrote an `eql_v4` block ORE +term before this one: the block ORE types were not producible. So a v4 +`_ord_ore` or `text_search_ore` row holding an 8-bit term exists only where +it was written by hand, or with cipherstash-client's ORE indexer, against +the earlier `eql_v4` bundle, which compared such terms. If you have such +rows, or query operands or `eql_v4.between` ranges built that way, they are +refused by this release's comparator with the first error above. They are +never ordered wrongly: a refused term is an error, not a result. + +**What to do.** Where such rows exist, write the affected columns again +through Stack Encrypt, which derives the 6-bit term for every block ORE +type, and build every block ORE query and range operand through it as well. +Do this before creating or rebuilding a functional index over +`eql_v4.ord_term_ore(col)`: an index build compares every row, so one +8-bit term in the table fails the build. Installs that never held an +`eql_v4` block ORE term, and installs whose block ORE domains are disabled +(no operator class), have nothing to do. + +**Verification.** + +```sql +-- The v4 bundle names its block ORE objects by the scheme: +SELECT p.proname +FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname = 'eql_v4_internal' AND p.proname LIKE '%ore_block%' +ORDER BY 1; -- ore_block_v2 names only; no ore_block_256 + +-- Every stored term in a block ORE column is one the comparator accepts: +-- this compares each row's term with itself and raises on a term it refuses. +SELECT count(*) FROM users WHERE age <= age; -- equals count(*), no error +``` + +## Verification checklist + +- [ ] The `eql_v4` install reports `3.1.0` (`SELECT obj_description('eql_v4'::regnamespace, 'pg_namespace')`). +- [ ] A one-term `eql_v4.query_text_search` operand casts (U-001). +- [ ] `eql_v4_internal` lists `ore_block_v2` objects and no `ore_block_256` (U-002). +- [ ] Every `_ord_ore` / `text_search_ore` column passes the self-comparison scan (U-002). +- [ ] Grants and functional indexes have been re-applied after the install + ([U-002 in the 3.0.5 guide](./v3.0.5.md#u-002-grants-and-dependent-objects-do-not-survive-an-install)). + +## Rollback + +Installing the 3.0.6 bundle reverses both notes at the SQL, and neither +reversal misreads data: + +- U-001: the 3.0.6 CHECK requires every term again, so a one-term query is + refused by the domain (SQLSTATE `23514`) rather than answered. +- U-002: the 3.0.6 v4 comparator expects the 8-bit layout, so a 6-bit term + written under 3.1.0 is refused as malformed, exactly as the 8-bit terms are + under 3.1.0. Rolling back a database that holds 6-bit terms therefore + means rewriting those columns too, with a Stack Encrypt that derives the + 8-bit term, and no release does. Decide before writing block ORE rows under + 3.1.0 whether a rollback must stay possible; if it must, hold off on the + block ORE domains and order through the `_ord` / `_ord_ope` domains, whose + terms this release does not change. + +Any install, in either direction, re-runs `DROP SCHEMA … CASCADE`, so grants +and dependent objects need re-applying afterwards. + +## See also + +- [Upgrading to EQL 3.0.5](./v3.0.5.md) — grants and dependent objects do + not survive an install. +- [Upgrading to EQL 3.0](./v3.0.md) — the 3.0 line, including why a + non-superuser install disables the block ORE domains. +- [Database indexes](../reference/database-indexes.md) — the index recipes, + and which need the ORE operator class. diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index a13b0fbc5..ecf7aca2e 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -340,7 +340,7 @@ psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -f release/cipherstash-encrypt- # `--ignored` selects by attribute, and cargo exits 0 on an empty selection, so # a test that loses its `#[ignore]` or is renamed would leave this task passing # on nothing. List what the filter selects before running it. -for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range v4_text_search; do +for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range v4_text_search v4_ore_block; do listing=$(cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --list | sed -n 's/: test$//p') [ -n "$listing" ] || { echo "error: '--ignored' selects no test in $suite; the PostgreSQL test has gone missing." >&2; exit 1; } cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored diff --git a/packages/eql/src/v3/common.sql b/packages/eql/src/v3/common.sql index 56c0edc33..4097ec4f2 100644 --- a/packages/eql/src/v3/common.sql +++ b/packages/eql/src/v3/common.sql @@ -19,7 +19,7 @@ --! @note Returns NULL if input is JSON null --! @note Each array element is hex-decoded to bytea --! @note plpgsql, not `LANGUAGE sql` (issue #353). This helper's ONLY caller ---! chain is `ore_block_256(val)` -> `jsonb_array_to_ore_block_256(val)` — +--! chain is `{{ore_block}}(val)` -> `jsonb_array_to_{{ore_block}}(val)` — --! both reached exclusively from plpgsql and btree operator-class support --! contexts, where SQL functions can NEVER be inlined and instead pay the --! per-call SQL-function executor (measured 3.5x the per-call cost of the @@ -49,7 +49,7 @@ $$ LANGUAGE plpgsql; --! @internal Keep the inline-critical marker so the post-install --! pin_search_path pass leaves this unpinned: a `SET search_path` clause on a --! plpgsql function forces per-call configuration switching — measurable on a ---! helper invoked per compared value in the ore_block_256 opclass hot path. +--! helper invoked per compared value in the {{ore_block}} opclass hot path. --! It takes a bare `jsonb` arg (not a jsonb-backed encrypted DOMAIN), so the --! structural skip in tasks/pin_search_path_v3.sql does not recognise it; --! this marker is the documented manual opt-in. diff --git a/packages/eql/src/v3/lint/lints.sql b/packages/eql/src/v3/lint/lints.sql index 1e87deade..54e798459 100644 --- a/packages/eql/src/v3/lint/lints.sql +++ b/packages/eql/src/v3/lint/lints.sql @@ -19,7 +19,7 @@ --! (or `pg_get_functiondef`); tracked as a follow-up. --! --! Operators on `{{prefix}}` types (the jsonb-backed encrypted-domain families and ---! the SEM index-term type `{{prefix}}_internal.ore_block_256`) whose +--! the SEM index-term type `{{prefix}}_internal.{{ore_block}}`) whose --! implementation functions fail any of these rules silently fall back to seq --! scan when the documented functional indexes (`{{prefix}}.eq_term(col)`, --! `{{prefix}}.ord_term(col)`) are in place. This lint surfaces every such case. @@ -58,7 +58,7 @@ --! functional index on the extractor instead. --! `schema_placement` — a naked composite or enum TYPE lives in the --! public `{{prefix}}` schema. Internal index-term ---! types (e.g. `ore_block_256_term`) belong in +--! types (e.g. `{{ore_block}}_term`) belong in --! `{{prefix}}_internal`; a composite/enum in --! `{{prefix}}` clutters the Supabase Table Builder --! type picker, which the schema split exists to diff --git a/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql index c2856dba4..aedbcafc2 100644 --- a/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/bigint/bigint_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_bigint_ord_ore. --! @param a public.{{prefix}}_bigint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_bigint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_bigint_ord_ore. --! @param a public.{{prefix}}_bigint_ord_ore diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql index c1dec7671..8aa94e2f8 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_bigint_ord_ore. --! @param a {{prefix}}.query_bigint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_bigint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_bigint_ord_ore. --! @param a public.{{prefix}}_bigint_ord_ore diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql index 0d9edb9c8..418810bc8 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_bigint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_bigint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_bigint_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_bigint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_bigint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_bigint_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_bigint_ord`, `public.{{prefix}}_bigint_ord_ope`) diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql index 37c422c54..476d4f9c4 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql b/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql index 676c12d33..6c7da4429 100644 --- a/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/date/date_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_date_ord_ore. --! @param a public.{{prefix}}_date_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_date_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_date_ord_ore. --! @param a public.{{prefix}}_date_ord_ore diff --git a/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql b/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql index c06aebf87..5150c1f40 100644 --- a/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/date/query_date_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_date_ord_ore. --! @param a {{prefix}}.query_date_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_date_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_date_ord_ore. --! @param a public.{{prefix}}_date_ord_ore diff --git a/packages/eql/src/v3/scalars/date/query_date_range_functions.sql b/packages/eql/src/v3/scalars/date/query_date_range_functions.sql index 840e33f1b..86a512a43 100644 --- a/packages/eql/src/v3/scalars/date/query_date_range_functions.sql +++ b/packages/eql/src/v3/scalars/date/query_date_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_date_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_date_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_date_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_date_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_date_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_date_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_date_ord`, `public.{{prefix}}_date_ord_ope`) diff --git a/packages/eql/src/v3/scalars/date/query_date_range_types.sql b/packages/eql/src/v3/scalars/date/query_date_range_types.sql index 31ec74c6a..0dcce4acc 100644 --- a/packages/eql/src/v3/scalars/date/query_date_range_types.sql +++ b/packages/eql/src/v3/scalars/date/query_date_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql b/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql index 8cdec0b7f..0c7bdd2a2 100644 --- a/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/double/double_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_double_ord_ore. --! @param a public.{{prefix}}_double_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_double_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_double_ord_ore. --! @param a public.{{prefix}}_double_ord_ore diff --git a/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql b/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql index e7e95f291..fe4943c20 100644 --- a/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/double/query_double_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_double_ord_ore. --! @param a {{prefix}}.query_double_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_double_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_double_ord_ore. --! @param a public.{{prefix}}_double_ord_ore diff --git a/packages/eql/src/v3/scalars/double/query_double_range_functions.sql b/packages/eql/src/v3/scalars/double/query_double_range_functions.sql index 8aa8ed9eb..d5282f6b2 100644 --- a/packages/eql/src/v3/scalars/double/query_double_range_functions.sql +++ b/packages/eql/src/v3/scalars/double/query_double_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_double_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_double_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_double_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_double_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_double_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_double_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_double_ord`, `public.{{prefix}}_double_ord_ope`) diff --git a/packages/eql/src/v3/scalars/double/query_double_range_types.sql b/packages/eql/src/v3/scalars/double/query_double_range_types.sql index 4939fa339..5603183ee 100644 --- a/packages/eql/src/v3/scalars/double/query_double_range_types.sql +++ b/packages/eql/src/v3/scalars/double/query_double_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql b/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql index 4bfd5d2eb..cfbb1ecbd 100644 --- a/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/integer/integer_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_integer_ord_ore. --! @param a public.{{prefix}}_integer_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_integer_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_integer_ord_ore. --! @param a public.{{prefix}}_integer_ord_ore diff --git a/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql b/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql index 3578cfb1a..824fcb264 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_integer_ord_ore. --! @param a {{prefix}}.query_integer_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_integer_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_integer_ord_ore. --! @param a public.{{prefix}}_integer_ord_ore diff --git a/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql b/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql index d05e5c03a..68924936e 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_integer_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_integer_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_integer_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_integer_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_integer_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_integer_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_integer_ord`, `public.{{prefix}}_integer_ord_ope`) diff --git a/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql b/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql index c32c1be6c..5c9fe2cee 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql b/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql index 748793ea5..519252c7b 100644 --- a/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/numeric/numeric_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_numeric_ord_ore. --! @param a public.{{prefix}}_numeric_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_numeric_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_numeric_ord_ore. --! @param a public.{{prefix}}_numeric_ord_ore diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql index 092a57f9c..3426a4055 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_numeric_ord_ore. --! @param a {{prefix}}.query_numeric_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_numeric_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_numeric_ord_ore. --! @param a public.{{prefix}}_numeric_ord_ore diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql index 193f9763b..60ac2d189 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_numeric_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_numeric_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_numeric_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_numeric_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_numeric_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_numeric_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_numeric_ord`, `public.{{prefix}}_numeric_ord_ope`) diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql index 2bd28ed5b..2e47c91db 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/ore_fallback.sql b/packages/eql/src/v3/scalars/ore_fallback.sql index ce56a258a..6a699836f 100644 --- a/packages/eql/src/v3/scalars/ore_fallback.sql +++ b/packages/eql/src/v3/scalars/ore_fallback.sql @@ -25,7 +25,7 @@ --! --! Runs after the DO block in src/v3/sem/ore_block_256/operator_class.sql, --! which ATTEMPTS to create the default btree operator class for ---! {{prefix}}_internal.ore_block_256 and skips it on insufficient_privilege +--! {{prefix}}_internal.{{ore_block}} and skips it on insufficient_privilege --! (CREATE OPERATOR CLASS requires superuser; managed platforms — cloud --! Supabase and most hosted Postgres — run the installer as a non-superuser --! role). When the class was created, this file is a no-op. @@ -62,7 +62,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql b/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql index db512dd9a..40e483688 100644 --- a/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/real/query_real_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_real_ord_ore. --! @param a {{prefix}}.query_real_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_real_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_real_ord_ore. --! @param a public.{{prefix}}_real_ord_ore diff --git a/packages/eql/src/v3/scalars/real/query_real_range_functions.sql b/packages/eql/src/v3/scalars/real/query_real_range_functions.sql index 875251fd6..c1a74e2ca 100644 --- a/packages/eql/src/v3/scalars/real/query_real_range_functions.sql +++ b/packages/eql/src/v3/scalars/real/query_real_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_real_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_real_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_real_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_real_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_real_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_real_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_real_ord`, `public.{{prefix}}_real_ord_ope`) diff --git a/packages/eql/src/v3/scalars/real/query_real_range_types.sql b/packages/eql/src/v3/scalars/real/query_real_range_types.sql index b9508d140..e79669906 100644 --- a/packages/eql/src/v3/scalars/real/query_real_range_types.sql +++ b/packages/eql/src/v3/scalars/real/query_real_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql b/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql index c88e2b5b9..44da6bc7f 100644 --- a/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/real/real_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_real_ord_ore. --! @param a public.{{prefix}}_real_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_real_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_real_ord_ore. --! @param a public.{{prefix}}_real_ord_ore diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql index 8d898cf4a..db35558c1 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_smallint_ord_ore. --! @param a {{prefix}}.query_smallint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_smallint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_smallint_ord_ore. --! @param a public.{{prefix}}_smallint_ord_ore diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql index 2ec4f1809..df1dffbe9 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_smallint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_smallint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_smallint_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_smallint_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_smallint_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_smallint_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_smallint_ord`, `public.{{prefix}}_smallint_ord_ope`) diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql index 74ec90df4..bf467bef2 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql b/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql index 4940f690f..c01347c89 100644 --- a/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/smallint/smallint_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_smallint_ord_ore. --! @param a public.{{prefix}}_smallint_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_smallint_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_smallint_ord_ore. --! @param a public.{{prefix}}_smallint_ord_ore diff --git a/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql b/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql index f0f4b99f2..4295a7368 100644 --- a/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_ord_ore_functions.sql @@ -16,11 +16,11 @@ AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; --! @brief Index extractor for {{prefix}}.query_text_ord_ore. --! @param a {{prefix}}.query_text_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_text_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_text_ord_ore. --! @param a public.{{prefix}}_text_ord_ore diff --git a/packages/eql/src/v3/scalars/text/query_text_range_functions.sql b/packages/eql/src/v3/scalars/text/query_text_range_functions.sql index 34735635d..02c5b9a9b 100644 --- a/packages/eql/src/v3/scalars/text/query_text_range_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_range_functions.sql @@ -23,14 +23,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_text_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_text_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_text_range_ore, as a block ORE term. --! @@ -40,14 +40,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_text_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_text_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_text_range_ope, as a CLLW OPE term. --! @@ -99,7 +99,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_text_ord`, `public.{{prefix}}_text_ord_ope`, `public.{{prefix}}_text_search`) @@ -134,7 +134,7 @@ AS $$ SELECT {{prefix}}.ord_term_ore(a) >= {{prefix}}_internal.lo(r) AND {{prefi --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_text_ord`, `public.{{prefix}}_text_ord_ope`, `public.{{prefix}}_text_search`) diff --git a/packages/eql/src/v3/scalars/text/query_text_range_types.sql b/packages/eql/src/v3/scalars/text/query_text_range_types.sql index 4b1fb707f..caaf084cf 100644 --- a/packages/eql/src/v3/scalars/text/query_text_range_types.sql +++ b/packages/eql/src/v3/scalars/text/query_text_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql b/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql index 44295168d..2815a05b3 100644 --- a/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql @@ -25,18 +25,18 @@ AS $$ SELECT {{prefix}}_internal.hmac_256({{prefix}}_internal.require_query_term --! @brief Index extractor for {{prefix}}.query_text_search_ore. --! @param a {{prefix}}.query_text_search_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} -- {{#v4}} --! @throws When the operand carries no `ob` term, the term <, <=, > and >= compare. -- {{/v4}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_text_search_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -- {{#v3}} -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; -- {{/v3}} -- {{#v4}} -AS $$ SELECT {{prefix}}_internal.ore_block_256({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'ob', '<, <=, > and >=')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'ob', '<, <=, > and >=')) $$; -- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search_ore. diff --git a/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql b/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql index d61d16612..74f3e200b 100644 --- a/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/text_ord_ore_functions.sql @@ -19,11 +19,11 @@ AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; --! @brief Index extractor for public.{{prefix}}_text_ord_ore. --! @param a public.{{prefix}}_text_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_text_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_text_ord_ore. --! @param a public.{{prefix}}_text_ord_ore diff --git a/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql b/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql index fb3eaa35d..c15e31063 100644 --- a/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/text_search_ore_functions.sql @@ -20,11 +20,11 @@ AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; --! @brief Index extractor for public.{{prefix}}_text_search_ore. --! @param a public.{{prefix}}_text_search_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_text_search_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Index extractor for public.{{prefix}}_text_search_ore. --! @param a public.{{prefix}}_text_search_ore diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql index b081475a3..5b583178c 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_ord_ore_functions.sql @@ -8,11 +8,11 @@ --! @brief Index extractor for {{prefix}}.query_timestamp_ord_ore. --! @param a {{prefix}}.query_timestamp_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_timestamp_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for {{prefix}}.query_timestamp_ord_ore. --! @param a public.{{prefix}}_timestamp_ord_ore diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql index 6c1a95c89..cab511249 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_functions.sql @@ -21,14 +21,14 @@ --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_timestamp_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The low bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The low bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.lo(r {{prefix}}.query_timestamp_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'lo')) $$; --! @brief The high bound of a {{prefix}}.query_timestamp_range_ore, as a block ORE term. --! @@ -38,14 +38,14 @@ AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb --! the public `{{prefix}}` surface. --! --! @param r {{prefix}}.query_timestamp_range_ore Range query operand ---! @return {{prefix}}_internal.ore_block_256 The high bound's term (included in the range), or NULL for a NULL range +--! @return {{prefix}}_internal.{{ore_block}} The high bound's term (included in the range), or NULL for a NULL range --! @note IMMUTABLE and STRICT. It is evaluated once per query when --! `{{prefix}}.between` is served by an index, since its argument is the --! query's parameter, not a column. CREATE FUNCTION {{prefix}}_internal.hi(r {{prefix}}.query_timestamp_range_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(jsonb_build_object('ob', r::jsonb -> 'hi')) $$; --! @brief The low bound of a {{prefix}}.query_timestamp_range_ope, as a CLLW OPE term. --! @@ -97,7 +97,7 @@ AS $$ SELECT {{prefix}}_internal.ope_cllw(jsonb_build_object('op', r::jsonb -> ' --! stop inlining. Inlined, the planner sees two comparisons on --! `{{prefix}}.ord_term_ore(a)`, which a btree index on that expression --! serves as one range scan. For this ORE range that index needs the ---! default btree operator class on {{prefix}}_internal.ore_block_256, which the +--! default btree operator class on {{prefix}}_internal.{{ore_block}}, which the --! installer creates only with superuser privileges. Without it (on Supabase, --! for instance) the installer disables the ORE column domains and this ORE range, --! so use an OPE column (`public.{{prefix}}_timestamp_ord`, `public.{{prefix}}_timestamp_ord_ope`) diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql index 795b8f267..02a882baa 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql @@ -83,7 +83,7 @@ BEGIN END $$; --- Without the default btree operator class on {{prefix}}_internal.ore_block_256 +-- Without the default btree operator class on {{prefix}}_internal.{{ore_block}} -- (the installer skips it on insufficient_privilege), ore_fallback.sql has -- created {{prefix}}_internal.ore_domain_unavailable and poisoned the -- single-bound ORE domains. It cannot reach this ORE range, which exists in @@ -96,7 +96,7 @@ BEGIN JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = '{{prefix}}_internal.ore_block_256'::pg_catalog.regtype + AND c.opcintype = '{{prefix}}_internal.{{ore_block}}'::pg_catalog.regtype ) THEN RETURN; END IF; diff --git a/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql b/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql index 6a83371f2..1727ad6da 100644 --- a/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql +++ b/packages/eql/src/v3/scalars/timestamp/timestamp_ord_ore_functions.sql @@ -10,11 +10,11 @@ --! @brief Index extractor for public.{{prefix}}_timestamp_ord_ore. --! @param a public.{{prefix}}_timestamp_ord_ore ---! @return {{prefix}}_internal.ore_block_256 +--! @return {{prefix}}_internal.{{ore_block}} CREATE FUNCTION {{prefix}}.ord_term_ore(a public.{{prefix}}_timestamp_ord_ore) -RETURNS {{prefix}}_internal.ore_block_256 +RETURNS {{prefix}}_internal.{{ore_block}} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +AS $$ SELECT {{prefix}}_internal.{{ore_block}}(a::jsonb) $$; --! @brief Operator wrapper for public.{{prefix}}_timestamp_ord_ore. --! @param a public.{{prefix}}_timestamp_ord_ore diff --git a/packages/eql/src/v3/schema.sql b/packages/eql/src/v3/schema.sql index d82dad656..bddd822bb 100644 --- a/packages/eql/src/v3/schema.sql +++ b/packages/eql/src/v3/schema.sql @@ -12,7 +12,7 @@ --! (Supabase/PostgREST calls functions, not operators) invoke them by name. --! {{prefix}}_internal houses INTERNAL implementation objects only: the --! searchable-encrypted-metadata (SEM) index-term types ---! ({{prefix}}_internal.hmac_256, {{prefix}}_internal.ore_block_256) and their support +--! ({{prefix}}_internal.hmac_256, {{prefix}}_internal.{{ore_block}}) and their support --! functions, the unsupported-operator blockers (which only raise), and the --! aggregate state functions. Together the two schemas are self-contained — --! they own every type they need and have no runtime dependency on another EQL diff --git a/packages/eql/src/v3/sem/bloom_filter/functions.sql b/packages/eql/src/v3/sem/bloom_filter/functions.sql index db4db5202..47e2295f7 100644 --- a/packages/eql/src/v3/sem/bloom_filter/functions.sql +++ b/packages/eql/src/v3/sem/bloom_filter/functions.sql @@ -16,7 +16,7 @@ --! @return boolean True when the `bf` key is present and non-null. --! --! @internal Defined for parity with the {{prefix}} SEM index-term predicates ---! (`has_hmac_256` / `has_ore_block_256`); it is not currently called by +--! (`has_hmac_256` / `has_{{ore_block}}`); it is not currently called by --! the extractor below, which gates on value-shape inline, nor by the generated --! domain CHECK, which tests `bf` presence via the envelope-key skeleton. Kept --! as the canonical presence test for callers that need one. diff --git a/packages/eql/src/v3/sem/ope_cllw/types.sql b/packages/eql/src/v3/sem/ope_cllw/types.sql index b27c193da..381696fc0 100644 --- a/packages/eql/src/v3/sem/ope_cllw/types.sql +++ b/packages/eql/src/v3/sem/ope_cllw/types.sql @@ -15,10 +15,10 @@ --! same pattern as {{prefix}}_internal.hmac_256 over text). That keeps the whole --! comparison chain inlinable, so a functional btree index on --! `{{prefix}}.ord_term(col)` engages structurally for the `_ord` / `_ord_ope` ---! domains' comparison operators. Contrast {{prefix}}_internal.ore_block_256 (`ob`), +--! domains' comparison operators. Contrast {{prefix}}_internal.{{ore_block}} (`ob`), --! the block-ORE term behind the `_ord_ore` escape hatch, compared by a custom --! N-block protocol. --! --! @note Transient type used only during query execution. ---! @see {{prefix}}_internal.ore_block_256 +--! @see {{prefix}}_internal.{{ore_block}} CREATE DOMAIN {{prefix}}_internal.ope_cllw AS bytea; diff --git a/packages/eql/src/v3/sem/ore_block_256/functions.sql b/packages/eql/src/v3/sem/ore_block_256/functions.sql index 1d6108e18..b77a4e48c 100644 --- a/packages/eql/src/v3/sem/ore_block_256/functions.sql +++ b/packages/eql/src/v3/sem/ore_block_256/functions.sql @@ -16,15 +16,15 @@ --! @brief Convert JSONB array to ORE block composite type --! @internal --! @param val jsonb Array of hex-encoded ORE block terms ---! @return {{prefix}}_internal.ore_block_256 ORE block composite, or NULL if input is null +--! @return {{prefix}}_internal.{{ore_block}} ORE block composite, or NULL if input is null --! @note plpgsql, not `LANGUAGE sql` (issue #353). The sole caller ---! (`ore_block_256`) is itself plpgsql, so this function is NEVER reached +--! (`{{ore_block}}`) is itself plpgsql, so this function is NEVER reached --! from an inlinable context — as `LANGUAGE sql` it paid the per-call --! SQL-function executor on every compared value in the opclass hot path --! (measured: +43% on ORE ordered scans vs the plpgsql form). The --! non-array guard preserves the v3 behaviour (returns NULL for a --! non-array scalar; the v2 plpgsql original raised); the caller only ---! reaches this when `has_ore_block_256(val)` is true, which requires +--! reaches this when `has_{{ore_block}}(val)` is true, which requires --! `val->'ob'` to be a JSON array, so that branch stays unreachable. --! An empty array (`ob: []`, what encrypting the empty string `""` produces) --! yields a non-NULL composite with an EMPTY `terms` array — NOT NULL terms. @@ -33,17 +33,17 @@ --! drops out of ordered queries). An empty array instead engages the --! comparator's `cardinality = 0` guard, which sorts empty BEFORE every --! non-empty term. See issue #262 (pinned by T7). -CREATE FUNCTION {{prefix}}_internal.jsonb_array_to_ore_block_256(val jsonb) -RETURNS {{prefix}}_internal.ore_block_256 +CREATE FUNCTION {{prefix}}_internal.jsonb_array_to_{{ore_block}}(val jsonb) +RETURNS {{prefix}}_internal.{{ore_block}} IMMUTABLE AS $$ DECLARE - terms {{prefix}}_internal.ore_block_256_term[]; + terms {{prefix}}_internal.{{ore_block}}_term[]; BEGIN IF val IS NULL OR jsonb_typeof(val) != 'array' THEN RETURN NULL; END IF; - SELECT array_agg(ROW(b)::{{prefix}}_internal.ore_block_256_term) + SELECT array_agg(ROW(b)::{{prefix}}_internal.{{ore_block}}_term) INTO terms FROM unnest({{prefix}}_internal.jsonb_array_to_bytea_array(val)) AS b; -- plpgsql pitfall: `SELECT INTO ` assigns the @@ -51,35 +51,35 @@ BEGIN -- constructor directly instead. The COALESCE stays load-bearing for the -- empty-`ob` case (issue #262): array_agg over zero rows yields NULL, and -- the comparator needs an EMPTY terms array, not NULL terms. - RETURN ROW(COALESCE(terms, ARRAY[]::{{prefix}}_internal.ore_block_256_term[]))::{{prefix}}_internal.ore_block_256; + RETURN ROW(COALESCE(terms, ARRAY[]::{{prefix}}_internal.{{ore_block}}_term[]))::{{prefix}}_internal.{{ore_block}}; END; $$ LANGUAGE plpgsql; --! @internal Keep the inline-critical marker so the post-install --! pin_search_path pass leaves this unpinned: a `SET search_path` clause on a --! plpgsql function forces per-call configuration switching — measurable on a ---! helper invoked per compared value in the ore_block_256 opclass hot path. +--! helper invoked per compared value in the {{ore_block}} opclass hot path. --! It takes a bare `jsonb` arg (not a jsonb-backed encrypted DOMAIN), so the --! structural skip in tasks/pin_search_path_v3.sql does not recognise it; --! this marker is the documented manual opt-in. -COMMENT ON FUNCTION {{prefix}}_internal.jsonb_array_to_ore_block_256(jsonb) IS +COMMENT ON FUNCTION {{prefix}}_internal.jsonb_array_to_{{ore_block}}(jsonb) IS 'eql-inline-critical: per-encrypted-value ORE opclass-path helper; must stay unpinned (SET search_path adds per-call overhead)'; --! @brief Extract ORE block index term from JSONB payload --! @param val jsonb containing encrypted EQL payload ---! @return {{prefix}}_internal.ore_block_256 ORE block index term +--! @return {{prefix}}_internal.{{ore_block}} ORE block index term --! @throws Exception if 'ob' field is missing -CREATE FUNCTION {{prefix}}_internal.ore_block_256(val jsonb) - RETURNS {{prefix}}_internal.ore_block_256 +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}(val jsonb) + RETURNS {{prefix}}_internal.{{ore_block}} IMMUTABLE STRICT PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN -- Declared STRICT: PostgreSQL returns NULL for a NULL argument without -- entering the body, so no explicit `val IS NULL` guard is needed. - IF {{prefix}}_internal.has_ore_block_256(val) THEN - RETURN {{prefix}}_internal.jsonb_array_to_ore_block_256(val->'ob'); + IF {{prefix}}_internal.has_{{ore_block}}(val) THEN + RETURN {{prefix}}_internal.jsonb_array_to_{{ore_block}}(val->'ob'); END IF; RAISE 'Expected an ore index (ob) value in json: %', val; END; @@ -91,13 +91,13 @@ $$ LANGUAGE plpgsql; --! @return boolean True only if the 'ob' field is present and is a JSON array --! @note A well-formed ORE index term is always a JSON array of block terms, so --! this guard treats a present-but-non-array `ob` (a scalar or object) as ---! absent. That makes the extractor `ore_block_256(val)` RAISE on a +--! absent. That makes the extractor `{{ore_block}}(val)` RAISE on a --! structurally invalid `ob` payload at the boundary instead of silently ---! degrading it to a NULL index term in `jsonb_array_to_ore_block_256`. The +--! degrading it to a NULL index term in `jsonb_array_to_{{ore_block}}`. The --! previous `val ->> 'ob' IS NOT NULL` form stringified scalars/objects and so --! reported them as present. `{}` (absent `ob`) and `{"ob": null}` (JSON null) --! both remain `false`. -CREATE FUNCTION {{prefix}}_internal.has_ore_block_256(val jsonb) +CREATE FUNCTION {{prefix}}_internal.has_{{ore_block}}(val jsonb) RETURNS boolean IMMUTABLE STRICT PARALLEL SAFE SET search_path = pg_catalog, extensions, public @@ -108,20 +108,21 @@ AS $$ $$ LANGUAGE plpgsql; +-- {{#v3}} --! @brief Compare two ORE block terms using cryptographic comparison --! @internal ---! @param a {{prefix}}_internal.ore_block_256_term First ORE term ---! @param b {{prefix}}_internal.ore_block_256_term Second ORE term +--! @param a {{prefix}}_internal.{{ore_block}}_term First ORE term +--! @param b {{prefix}}_internal.{{ore_block}}_term Second ORE term --! @return integer -1 if a < b, 0 if a = b, 1 if a > b --! @throws Exception if ciphertexts are different lengths ---! @note Marked `IMMUTABLE` (the three `compare_ore_block_256_term(s)` +--! @note Marked `IMMUTABLE` (the three `compare_{{ore_block}}_term(s)` --! overloads all are). This deliberately diverges from the v2 originals, --! which carry no volatility marker and so default to `VOLATILE`. The --! comparison is deterministic — its only crypto call, pgcrypto `encrypt()`, --! is itself `IMMUTABLE STRICT PARALLEL SAFE` — so `IMMUTABLE` lets the --! planner fold/cache these in ordering and index contexts. NOT `STRICT`: --! the NULL-handling branches below are load-bearing for the array overload. -CREATE FUNCTION {{prefix}}_internal.compare_ore_block_256_term(a {{prefix}}_internal.ore_block_256_term, b {{prefix}}_internal.ore_block_256_term) +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_term(a {{prefix}}_internal.{{ore_block}}_term, b {{prefix}}_internal.{{ore_block}}_term) RETURNS integer IMMUTABLE SET search_path = pg_catalog, extensions, public @@ -221,10 +222,10 @@ $$ LANGUAGE plpgsql; --! @brief Compare arrays of ORE block terms recursively --! @internal ---! @param a {{prefix}}_internal.ore_block_256_term[] First array ---! @param b {{prefix}}_internal.ore_block_256_term[] Second array +--! @param a {{prefix}}_internal.{{ore_block}}_term[] First array +--! @param b {{prefix}}_internal.{{ore_block}}_term[] Second array --! @return integer -1/0/1, or NULL if either array is NULL -CREATE FUNCTION {{prefix}}_internal.compare_ore_block_256_terms(a {{prefix}}_internal.ore_block_256_term[], b {{prefix}}_internal.ore_block_256_term[]) +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}_term[], b {{prefix}}_internal.{{ore_block}}_term[]) RETURNS integer IMMUTABLE SET search_path = pg_catalog, extensions, public @@ -248,28 +249,269 @@ AS $$ RETURN 1; END IF; - cmp_result := {{prefix}}_internal.compare_ore_block_256_term(a[1], b[1]); + cmp_result := {{prefix}}_internal.compare_{{ore_block}}_term(a[1], b[1]); IF cmp_result = 0 THEN - RETURN {{prefix}}_internal.compare_ore_block_256_terms(a[2:array_length(a,1)], b[2:array_length(b,1)]); + RETURN {{prefix}}_internal.compare_{{ore_block}}_terms(a[2:array_length(a,1)], b[2:array_length(b,1)]); END IF; RETURN cmp_result; END $$ LANGUAGE plpgsql; +-- {{/v3}} +-- {{#v4}} +--! @brief The 1-bit hash of a 6-bit block ORE comparison +--! @internal +--! @param f bytea The 16-byte left tag of the first differing block +--! @param nonce bytea The 16-byte nonce of the right-hand ciphertext +--! @return integer 0 or 1 +--! @note The BHKR sigma-MMO hash ore-rs's 6-bit schemes use: +--! `m = 2*f XOR nonce` (doubling in GF(2^128), big-endian, reduction +--! constant 0x87), then the low bit of byte 0 of `AES(PI_KEY, m) XOR m`, +--! where `PI_KEY` is the public, fixed AES key `ORE-rs.v2.H-pi.1`. The +--! doubling is done on two `bigint` halves with no branch on the data. Its +--! vectors are ore.rs's own (`tests/encryption/tests/v4_ore_block.rs`). +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_hash_bit(f bytea, nonce bytea) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE + SET search_path = pg_catalog, extensions, public +AS $$ + DECLARE + hi bigint := ('x' || encode(substr(f, 1, 8), 'hex'))::bit(64)::bigint; + lo bigint := ('x' || encode(substr(f, 9, 8), 'hex'))::bit(64)::bigint; + m bytea; + BEGIN + -- 2*f: shift the 128 bits left by one, carrying lo's top bit into hi, + -- and fold hi's top bit back in as 0x87. Then XOR the nonce. + m := int8send(((hi << 1) | (lo < 0)::int) + # ('x' || encode(substr(nonce, 1, 8), 'hex'))::bit(64)::bigint) + || int8send(((lo << 1) # (135 * (hi < 0)::int)) + # ('x' || encode(substr(nonce, 9, 8), 'hex'))::bit(64)::bigint); + RETURN get_bit(encrypt(m, 'ORE-rs.v2.H-pi.1'::bytea, 'aes-ecb/pad:none'), 0) + # get_bit(m, 0); + END; +$$ LANGUAGE plpgsql; + + +--! @brief The scheme of a 6-bit block ORE term, checking its shape +--! @internal +--! @param t bytea A serialised ore-rs ciphertext +--! @return integer 2 for the fixed-width 6-bit scheme, 3 for the chained one +--! @throws Exception if `t` is not a well-formed ciphertext of either: a +--! header of version 2, a block count of at least one (at most 14 for the +--! fixed-width scheme, at most 1366 for the chained one), a length of +--! exactly header + 25 bytes a block + a 16-byte nonce, a padding policy +--! the chained scheme writes with a block count it can produce, and every +--! permuted symbol below 64. The message names the length only, never the +--! bytes. +--! @note The checks ore-rs's comparators make before they compare; every +--! one is on the term's public structure. All of them are expressions, +--! with no sub-select, because this runs on both operands of every +--! comparison. +--! @note The chained bound is tighter than the wire format's 65,535 blocks. +--! 1366 blocks is a 1,024-byte string, the longest Stack Encrypt writes +--! (ore-rs's default ceiling, which counts the marker byte before text +--! and bytes). Comparing a term costs time in its block count, so a +--! stored term past the bound would slow every comparison against it. +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_scheme(t bytea) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE + SET search_path = pg_catalog, extensions, public +AS $$ + DECLARE + len integer := octet_length(t); + scheme integer; + header integer; + count integer; + max_count integer; + kind integer; + policy integer; + bucket integer; + admitted boolean; + BEGIN + IF len < 7 OR get_byte(t, 0) <> 2 OR get_byte(t, 1) NOT IN (2, 3) THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + scheme := get_byte(t, 1); + header := CASE scheme WHEN 2 THEN 4 ELSE 7 END; + count := get_byte(t, 2) * 256 + get_byte(t, 3); + max_count := CASE scheme WHEN 2 THEN 14 ELSE 1366 END; + IF count < 1 OR count > max_count OR len <> header + 25 * count + 16 THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + IF scheme = 3 THEN + -- The padding policy: none (0, 0), powers of two from a floor (1, n) + -- or one fixed length (2, n), and a block count it can produce. A + -- string of n bytes is ceil(8n / 6) blocks, which grows with n, so + -- the one length that could give `count` blocks is floor(6 count / 8). + kind := get_byte(t, 4); + policy := get_byte(t, 5) * 256 + get_byte(t, 6); + bucket := 6 * count / 8; + admitted := CASE kind + WHEN 0 THEN policy = 0 + WHEN 1 THEN policy <= 49151 AND ( + count = (8 * policy + 5) / 6 + OR (bucket > policy AND bucket & (bucket - 1) = 0 + AND count = (8 * bucket + 5) / 6)) + WHEN 2 THEN policy <= 49151 AND count = (8 * policy + 5) / 6 + ELSE false + END; + IF NOT admitted THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + END IF; + -- A byte is below 64 exactly when its high hex digit is 0 to 3. + IF encode(substr(t, header + 1, count), 'hex') !~ '^([0-3][0-9a-f])*$' THEN + RAISE EXCEPTION 'Malformed ORE block term: % bytes', len; + END IF; + RETURN scheme; + END; +$$ LANGUAGE plpgsql; + + +--! @brief Compare two 6-bit block ORE terms +--! @internal +--! @param a {{prefix}}_internal.{{ore_block}}_term First term: its left half is read +--! @param b {{prefix}}_internal.{{ore_block}}_term Second term: its right half is read +--! @return integer -1 if a < b, 0 if a = b, 1 if a > b +--! @throws Exception if either term is malformed +--! (`{{prefix}}_internal.{{ore_block}}_scheme`), if the two are of different +--! schemes, if two fixed-width terms have different block counts, or if +--! two chained terms were padded under different policies. A mismatched +--! pair raises rather than comparing to a meaningless answer. +--! @note Stack Encrypt writes ore-rs's 6-bit block ORE (ADR-0002): the +--! fixed-width scheme (wire header `02 02`) for integers and floats, and +--! the chained scheme (`02 03`, then a padding policy) for text. Both are +--! ore-rs's wire version 2, which names the type here (`{{ore_block}}`); +--! EQL v3's type of the same role compares cipherstash-client's 8-bit +--! blocks, a different format under the same `ob` key. +--! The layout after the header, for `count` blocks: `count` permuted +--! symbols, `count` 16-byte left tags, a 16-byte nonce, then `count` +--! 8-byte right blocks. Chained terms compare lexicographically: if the +--! shorter one's blocks all match, the shorter sorts first. +--! @note Constant operation count for operands of one shape (scheme and +--! block counts, which a term's length reveals anyway). One statement per +--! compared block evaluates both of its comparisons (integer `|`, which +--! does not short-circuit) and latches the first differing block with +--! arithmetic rather than an `IF`. The hash runs exactly once on every +--! path, equal terms included (on block 0, its result discarded), and the +--! result is selected arithmetically. PL/pgSQL is not a constant-time +--! environment: C-level effects remain (`memcmp` stopping inside 16 bytes, +--! the offset `substr` reads at). See `tests/timing/ore_block_v2/`. +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_term(a {{prefix}}_internal.{{ore_block}}_term, b {{prefix}}_internal.{{ore_block}}_term) + RETURNS integer + IMMUTABLE + SET search_path = pg_catalog, extensions, public +AS $$ + DECLARE + ab bytea; + bb bytea; + scheme integer; + header integer; + ca integer; -- a's block count + cb integer; -- b's block count + nonce_b integer; -- 0-based offset of b's nonce; its right blocks follow + latch integer := 0; -- 1 + the first differing block, 0 while none differs + first_diff integer; + indicator integer; + BEGIN + IF a IS NULL AND b IS NULL THEN + RETURN 0; + END IF; + + IF a IS NULL THEN + RETURN -1; + END IF; + + IF b IS NULL THEN + RETURN 1; + END IF; + + ab := a.bytes; + bb := b.bytes; + scheme := {{prefix}}_internal.{{ore_block}}_scheme(ab); + IF {{prefix}}_internal.{{ore_block}}_scheme(bb) <> scheme THEN + RAISE EXCEPTION 'ORE block terms of different schemes'; + END IF; + header := CASE scheme WHEN 2 THEN 4 ELSE 7 END; + ca := get_byte(ab, 2) * 256 + get_byte(ab, 3); + cb := get_byte(bb, 2) * 256 + get_byte(bb, 3); + IF scheme = 2 AND ca <> cb THEN + RAISE EXCEPTION 'Ciphertexts are different lengths'; + END IF; + IF scheme = 3 AND substr(ab, 5, 3) <> substr(bb, 5, 3) THEN + RAISE EXCEPTION 'ORE block terms padded under different policies'; + END IF; + nonce_b := header + 17 * cb; + + -- One statement per block, the same for every block. + FOR block IN 0..least(ca, cb) - 1 LOOP + latch := latch + (latch = 0)::int * (block + 1) * ( + (get_byte(ab, header + block) <> get_byte(bb, header + block))::int + | (substr(ab, header + ca + 16 * block + 1, 16) + <> substr(bb, header + cb + 16 * block + 1, 16))::int); + END LOOP; + + -- Equal prefixes read block 0. + first_diff := greatest(latch - 1, 0); + + -- One bit read and one hash on every path: bit `xt` of b's right block, + -- XOR the hash of a's left tag under b's nonce. 1 means a > b. + indicator := get_bit( + substr(bb, nonce_b + 17 + 8 * first_diff, 8), + get_byte(ab, header + first_diff)) + # {{prefix}}_internal.{{ore_block}}_hash_bit( + substr(ab, header + ca + 16 * first_diff + 1, 16), + substr(bb, nonce_b + 1, 16)); + + -- No difference: the shorter sorts first (equal lengths are equal). + -- Otherwise the indicator. No branch on the data. + RETURN (latch = 0)::int * sign(ca - cb)::integer + + (latch > 0)::int * (2 * indicator - 1); + END; +$$ LANGUAGE plpgsql; + + +--! @brief Compare two EQL v4 ORE block values +--! @internal +--! @param a {{prefix}}_internal.{{ore_block}}_term[] First value's terms +--! @param b {{prefix}}_internal.{{ore_block}}_term[] Second value's terms +--! @return integer -1/0/1, or NULL if either array is NULL +--! @throws Exception unless each value holds exactly one term: Stack +--! Encrypt writes one 6-bit block ORE term per value, text included, where +--! EQL v3's text held up to six 8-bit terms +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}_term[], b {{prefix}}_internal.{{ore_block}}_term[]) +RETURNS integer + IMMUTABLE + SET search_path = pg_catalog, extensions, public +AS $$ + BEGIN + IF a IS NULL OR b IS NULL THEN + RETURN NULL; + END IF; + + IF cardinality(a) <> 1 OR cardinality(b) <> 1 THEN + RAISE EXCEPTION 'An ORE block value holds exactly one term, not % and %', + cardinality(a), cardinality(b); + END IF; + + RETURN {{prefix}}_internal.compare_{{ore_block}}_term(a[1], b[1]); + END +$$ LANGUAGE plpgsql; +-- {{/v4}} --! @brief Compare ORE block composite types --! @internal ---! @param a {{prefix}}_internal.ore_block_256 First ORE block ---! @param b {{prefix}}_internal.ore_block_256 Second ORE block +--! @param a {{prefix}}_internal.{{ore_block}} First ORE block +--! @param b {{prefix}}_internal.{{ore_block}} Second ORE block --! @return integer -1/0/1 -CREATE FUNCTION {{prefix}}_internal.compare_ore_block_256_terms(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS integer IMMUTABLE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.compare_ore_block_256_terms(a.terms, b.terms); + RETURN {{prefix}}_internal.compare_{{ore_block}}_terms(a.terms, b.terms); END $$ LANGUAGE plpgsql; diff --git a/packages/eql/src/v3/sem/ore_block_256/operator_class.sql b/packages/eql/src/v3/sem/ore_block_256/operator_class.sql index b889c5cd9..c2a5f1bf6 100644 --- a/packages/eql/src/v3/sem/ore_block_256/operator_class.sql +++ b/packages/eql/src/v3/sem/ore_block_256/operator_class.sql @@ -4,7 +4,7 @@ -- REQUIRE: src/v3/sem/ore_block_256/operators.sql --! @file v3/sem/ore_block_256/operator_class.sql ---! @brief B-tree operator family + default class on {{prefix}}_internal.ore_block_256. +--! @brief B-tree operator family + default class on {{prefix}}_internal.{{ore_block}}. --! --! Gives the composite type its DEFAULT btree opclass so the recommended --! functional index `CREATE INDEX ON t ({{prefix}}.ord_term_ore(col))` engages without @@ -16,32 +16,32 @@ --! the installer as a non-superuser role, so the DO block below ATTEMPTS --! the creation and skips it on insufficient_privilege (SQLSTATE 42501), --! letting the single installer run everywhere. When the class is absent, ---! ORE ordered scans over {{prefix}}_internal.ore_block_256 are unavailable, +--! ORE ordered scans over {{prefix}}_internal.{{ore_block}} are unavailable, --! but the order-preserving (OPE) ordering domains — whose extractor --! return types carry a native btree opclass — still index without it. On --! superuser installs (self-managed Postgres, the SQLx test matrix) the --! class is created normally. Any non-privilege error still propagates. ---! @see {{prefix}}_internal.compare_ore_block_256_terms +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms DO $do$ BEGIN - EXECUTE 'CREATE OPERATOR FAMILY {{prefix}}_internal.ore_block_256_operator_family USING btree'; + EXECUTE 'CREATE OPERATOR FAMILY {{prefix}}_internal.{{ore_block}}_operator_family USING btree'; EXECUTE $ddl$ - CREATE OPERATOR CLASS {{prefix}}_internal.ore_block_256_operator_class - DEFAULT FOR TYPE {{prefix}}_internal.ore_block_256 - USING btree FAMILY {{prefix}}_internal.ore_block_256_operator_family AS + CREATE OPERATOR CLASS {{prefix}}_internal.{{ore_block}}_operator_class + DEFAULT FOR TYPE {{prefix}}_internal.{{ore_block}} + USING btree FAMILY {{prefix}}_internal.{{ore_block}}_operator_family AS OPERATOR 1 public.<, OPERATOR 2 public.<=, OPERATOR 3 public.=, OPERATOR 4 public.>=, OPERATOR 5 public.>, - FUNCTION 1 {{prefix}}_internal.compare_ore_block_256_terms(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) + FUNCTION 1 {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) $ddl$; - RAISE NOTICE 'EQL: created btree operator class {{prefix}}_internal.ore_block_256_operator_class'; + RAISE NOTICE 'EQL: created btree operator class {{prefix}}_internal.{{ore_block}}_operator_class'; EXCEPTION WHEN insufficient_privilege THEN - RAISE NOTICE 'EQL: skipped operator class {{prefix}}_internal.ore_block_256_operator_class (requires superuser); ORE ordered indexes on ore_block_256 unavailable, OPE ordering domains unaffected'; + RAISE NOTICE 'EQL: skipped operator class {{prefix}}_internal.{{ore_block}}_operator_class (requires superuser); ORE ordered indexes on {{ore_block}} unavailable, OPE ordering domains unaffected'; END; $do$; diff --git a/packages/eql/src/v3/sem/ore_block_256/operators.sql b/packages/eql/src/v3/sem/ore_block_256/operators.sql index cdd4c9df3..8a72f788f 100644 --- a/packages/eql/src/v3/sem/ore_block_256/operators.sql +++ b/packages/eql/src/v3/sem/ore_block_256/operators.sql @@ -3,7 +3,7 @@ -- REQUIRE: src/v3/sem/ore_block_256/functions.sql --! @file v3/sem/ore_block_256/operators.sql ---! @brief Comparison operators on {{prefix}}_internal.ore_block_256. +--! @brief Comparison operators on {{prefix}}_internal.{{ore_block}}. --! --! The six backing functions are inlinable single-statement SQL so the planner --! can fold the {{prefix}} comparison wrappers through to functional-index matching. @@ -11,97 +11,97 @@ --! @brief Equality backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the ORE blocks are equal --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_eq(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_eq(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) = 0 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) = 0 $$; --! @brief Not-equal backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the ORE blocks are not equal --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_neq(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_neq(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) <> 0 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) <> 0 $$; --! @brief Less-than backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is less than the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_lt(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_lt(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) = -1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) = -1 $$; --! @brief Less-than-or-equal backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is less than or equal to the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_lte(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_lte(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) != 1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) != 1 $$; --! @brief Greater-than backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is greater than the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_gt(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_gt(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) = 1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) = 1 $$; --! @brief Greater-than-or-equal backing function for ORE block types --! @internal --! ---! @param a {{prefix}}_internal.ore_block_256 Left operand ---! @param b {{prefix}}_internal.ore_block_256 Right operand +--! @param a {{prefix}}_internal.{{ore_block}} Left operand +--! @param b {{prefix}}_internal.{{ore_block}} Right operand --! @return boolean True if the left operand is greater than or equal to the right operand --! ---! @see {{prefix}}_internal.compare_ore_block_256_terms -CREATE FUNCTION {{prefix}}_internal.ore_block_256_gte(a {{prefix}}_internal.ore_block_256, b {{prefix}}_internal.ore_block_256) +--! @see {{prefix}}_internal.compare_{{ore_block}}_terms +CREATE FUNCTION {{prefix}}_internal.{{ore_block}}_gte(a {{prefix}}_internal.{{ore_block}}, b {{prefix}}_internal.{{ore_block}}) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.compare_ore_block_256_terms(a, b) != -1 + SELECT {{prefix}}_internal.compare_{{ore_block}}_terms(a, b) != -1 $$; @@ -112,9 +112,9 @@ $$; --! first time an ore_block equality is used as a join qual (e.g. via the inlined --! {{prefix}}_internal._ord_ore equality wrappers). CREATE OPERATOR public.= ( - FUNCTION={{prefix}}_internal.ore_block_256_eq, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_eq, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.=), NEGATOR = OPERATOR(public.<>), RESTRICT = eqsel, @@ -125,9 +125,9 @@ CREATE OPERATOR public.= ( --! @brief <> operator for ORE block types CREATE OPERATOR public.<> ( - FUNCTION={{prefix}}_internal.ore_block_256_neq, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_neq, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.<>), NEGATOR = OPERATOR(public.=), RESTRICT = neqsel, @@ -137,9 +137,9 @@ CREATE OPERATOR public.<> ( --! @brief > operator for ORE block types CREATE OPERATOR public.> ( - FUNCTION={{prefix}}_internal.ore_block_256_gt, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_gt, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.<), NEGATOR = OPERATOR(public.<=), RESTRICT = scalargtsel, @@ -148,9 +148,9 @@ CREATE OPERATOR public.> ( --! @brief < operator for ORE block types CREATE OPERATOR public.< ( - FUNCTION={{prefix}}_internal.ore_block_256_lt, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_lt, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.>), NEGATOR = OPERATOR(public.>=), RESTRICT = scalarltsel, @@ -159,9 +159,9 @@ CREATE OPERATOR public.< ( --! @brief <= operator for ORE block types CREATE OPERATOR public.<= ( - FUNCTION={{prefix}}_internal.ore_block_256_lte, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_lte, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.>=), NEGATOR = OPERATOR(public.>), RESTRICT = scalarlesel, @@ -170,9 +170,9 @@ CREATE OPERATOR public.<= ( --! @brief >= operator for ORE block types CREATE OPERATOR public.>= ( - FUNCTION={{prefix}}_internal.ore_block_256_gte, - LEFTARG={{prefix}}_internal.ore_block_256, - RIGHTARG={{prefix}}_internal.ore_block_256, + FUNCTION={{prefix}}_internal.{{ore_block}}_gte, + LEFTARG={{prefix}}_internal.{{ore_block}}, + RIGHTARG={{prefix}}_internal.{{ore_block}}, COMMUTATOR = OPERATOR(public.<=), NEGATOR = OPERATOR(public.<), RESTRICT = scalargesel, diff --git a/packages/eql/src/v3/sem/ore_block_256/types.sql b/packages/eql/src/v3/sem/ore_block_256/types.sql index b697ddaf6..f9996c611 100644 --- a/packages/eql/src/v3/sem/ore_block_256/types.sql +++ b/packages/eql/src/v3/sem/ore_block_256/types.sql @@ -10,7 +10,7 @@ --! --! Composite type representing a single ORE block term. Stores encrypted data --! as bytea that enables range comparisons without decryption. -CREATE TYPE {{prefix}}_internal.ore_block_256_term AS ( +CREATE TYPE {{prefix}}_internal.{{ore_block}}_term AS ( bytes bytea ); @@ -21,6 +21,6 @@ CREATE TYPE {{prefix}}_internal.ore_block_256_term AS ( --! in the 'ob' field of encrypted data payloads. --! --! @note Transient type used only during query execution. -CREATE TYPE {{prefix}}_internal.ore_block_256 AS ( - terms {{prefix}}_internal.ore_block_256_term[] +CREATE TYPE {{prefix}}_internal.{{ore_block}} AS ( + terms {{prefix}}_internal.{{ore_block}}_term[] ); diff --git a/packages/eql/tasks/pin_search_path_v3.sql b/packages/eql/tasks/pin_search_path_v3.sql index 8b5576617..593c6a699 100644 --- a/packages/eql/tasks/pin_search_path_v3.sql +++ b/packages/eql/tasks/pin_search_path_v3.sql @@ -12,7 +12,7 @@ --! satisfying Supabase splinter's `function_search_path_mutable` lint. --! --! @note A SET clause disables SQL-function inlining. The inline-critical SEM ---! helpers (ore_block_256_*, ope_cllw, hmac_256, bloom_filter over +--! helpers ({{ore_block}}_*, ope_cllw, hmac_256, bloom_filter over --! jsonb) and the encrypted-domain family (recognised structurally, --! including public user-column domains) are deliberately left --! unpinned. @@ -43,9 +43,9 @@ BEGIN WHERE n.nspname = ANY({{prefix}}_internal.owned_schemas()) AND ( (p.pronargs = 2 - AND p.proname IN ('ore_block_256_eq', 'ore_block_256_neq', - 'ore_block_256_lt', 'ore_block_256_lte', - 'ore_block_256_gt', 'ore_block_256_gte')) + AND p.proname IN ('{{ore_block}}_eq', '{{ore_block}}_neq', + '{{ore_block}}_lt', '{{ore_block}}_lte', + '{{ore_block}}_gt', '{{ore_block}}_gte')) -- The CLLW-OPE surface is the extractor alone: {{prefix}}_internal.ope_cllw is a -- domain over bytea (native comparison operators and btree opclass), -- so there are no ope-specific comparison functions to keep inlinable. diff --git a/packages/eql/tests/encryption/Cargo.toml b/packages/eql/tests/encryption/Cargo.toml index 939845ab4..6ee6ec13c 100644 --- a/packages/eql/tests/encryption/Cargo.toml +++ b/packages/eql/tests/encryption/Cargo.toml @@ -53,10 +53,6 @@ postgres = "0.19" # crates protect-ffi ships. Primitive dependencies such as blake3 are not # compared. # -# The EQL v4 range tests (tests/v4_range.rs) use it too: EQL compares -# block-ORE and CLLW-OPE terms, which Stack Encrypt does not derive for EQL -# yet, so cipherstash-client's `OreIndexer` / `OpeIndexer` write the order -# terms of their rows and ranges, offline over the same index key. cipherstash-client = "=0.42.3" # The tampering test decodes a stored body, changes one byte and re-encodes # it. The same line eql-bindings uses, so the lockfile gains no second copy. diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index 203bda902..523ea850e 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -375,15 +375,15 @@ mod given_a_name_the_engine_cannot_produce { name, ); } - let error = refused(targets::encrypt("TextOrdOre", &keyset, &email(), text("x"))); + let error = refused(targets::encrypt("DateOrdOre", &keyset, &email(), text("x"))); assert_eq!( error.to_string(), - "the engine cannot produce TextOrdOre yet: EQL stores block ORE terms and the engine \ - derives CLLW ORE (#1132)" + "the engine cannot produce DateOrdOre yet: the engine derives no order term for a date, \ + timestamp or decimal yet: how they are ordered is not decided (#1236)" ); // Refused by name before the label or the value is looked at. let error = refused(targets::encrypt( - "TextOrdOre", + "DateOrdOre", &keyset, &Label::new(["one"]).unwrap(), frame(None::), @@ -453,7 +453,7 @@ mod the_table { match term { Term::Hm => "eq", Term::Bloom => "match", - Term::Ore => "ore", + Term::Ore => "ore_block", Term::Ope => "ope", } } @@ -584,10 +584,10 @@ mod the_table { /// The types the engine produces in EQL v4 today: the storage and /// equality domains of every family whose plaintext eql-bindings names, - /// the OPE-ordered domains of those the engine derives an order term - /// for, and text match and search. + /// the ordered domains (OPE and block ORE) of those the engine derives + /// an order term for, and text match and search. #[test] - fn the_producible_types_are_the_ones_with_no_ore_and_a_rust_plaintext() { + fn the_producible_types_are_the_ones_with_a_rust_plaintext() { let producible: Vec<&str> = targets::targets() .iter() .filter(|t| t.producible) @@ -598,14 +598,17 @@ mod the_table { [ "Integer", "IntegerEq", + "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", "Smallint", "SmallintEq", + "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", "Bigint", "BigintEq", + "BigintOrdOre", "BigintOrd", "BigintOrdOpe", "Date", @@ -617,16 +620,20 @@ mod the_table { "Text", "TextEq", "TextMatch", + "TextOrdOre", "TextOrd", "TextOrdOpe", + "TextSearchOre", "TextSearch", "Boolean", "Real", "RealEq", + "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", + "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe" ], @@ -639,7 +646,6 @@ mod the_table { ("date" | "timestamp" | "numeric", "Ord" | "OrdOpe" | "OrdOre") => { "no order term for a date, timestamp or decimal yet" } - (_, "OrdOre" | "SearchOre") => "derives CLLW ORE (#1132)", (family, suffix) => panic!("{}: unexpected {family} {suffix}", row.name), }; assert!(reason.contains(expected), "{}: {reason}", row.name); diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs index 5174df470..1f5ec021d 100644 --- a/packages/eql/tests/encryption/tests/typed_queries.rs +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -318,24 +318,32 @@ fn the_typed_chain_answers_what_the_data_path_answers() { } } } - // The gap today: every producible ordered type, and text search, whose - // twin carries an order term among its three. A type leaves this list + // The gap today: every producible ordered type (OPE and block ORE), and + // text search, whose twins carry an order term among their three. A type + // leaves this list // when the typed chain assembles its twin, and its answers are then // held to the data path's above. assert_eq!( gap, [ + "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", + "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", + "BigintOrdOre", "BigintOrd", "BigintOrdOpe", + "TextOrdOre", "TextOrd", "TextOrdOpe", + "TextSearchOre", "TextSearch", + "RealOrdOre", "RealOrd", "RealOrdOpe", + "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", ] diff --git a/packages/eql/tests/encryption/tests/v4_order.rs b/packages/eql/tests/encryption/tests/v4_order.rs index dc3290840..d0567e2d0 100644 --- a/packages/eql/tests/encryption/tests/v4_order.rs +++ b/packages/eql/tests/encryption/tests/v4_order.rs @@ -97,8 +97,7 @@ fn the_ordered_producible_types_are_these() { let mut expected = ORDERED.to_vec(); expected.sort_unstable(); assert_eq!(ordered, expected); - // No producible type carries an ORE term: EQL's is block ORE, and the - // engine's is CLLW ORE. + // No type carries a CLLW ORE term: EQL's ORE term is block ORE. assert!(targets::targets() .iter() .filter(|t| t.producible) diff --git a/packages/eql/tests/encryption/tests/v4_ore_block.rs b/packages/eql/tests/encryption/tests/v4_ore_block.rs new file mode 100644 index 000000000..8cd02b70b --- /dev/null +++ b/packages/eql/tests/encryption/tests/v4_ore_block.rs @@ -0,0 +1,797 @@ +//! The block ORE EQL v4 types Stack Encrypt produces: `IntegerOrdOre`, +//! `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, `DoubleOrdOre`, +//! `TextOrdOre` and `TextSearchOre`, and EQL v4's comparator for their `ob` term. +//! +//! The term is ore-rs's 6-bit block ORE: the fixed-width scheme (wire +//! header `02 02`) for the integers and floats, the chained scheme (`02 03`, +//! text padded to a power-of-two length bucket of at least 16 bytes) for +//! text. Every term carries a random nonce, so the stored bytes differ on +//! each write and only the comparator orders them. +//! +//! Against an EQL v4 database these tests compare pairs of real ore-rs +//! ciphertexts with `eql_v4_internal.compare_ore_block_v2_term`, both ways +//! round, and require the answer to be the plaintext order and ore-rs's own +//! comparator's: equal values, a first difference in every block, shared +//! prefixes, and text in different padding buckets. They check the hash bit +//! against ore.rs's vectors, that malformed and mismatched terms raise, that +//! EQL admits the block counts ore-rs does up to 1366 blocks (the longest +//! term Stack Encrypt writes) and none past it, and that the domains answer +//! `<`, `=` and `ORDER BY` through a btree index. + +mod common; + +use std::cmp::Ordering; + +use eql_bindings::encryption::targets; +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::sem::{Orderable, OreBlockTerm}; +use stack_encrypt::{Encrypt, Label}; + +type Keyset<'a> = stack_encrypt::KeysetCipher<'a, common::Observed>; + +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() +} + +fn label() -> Label { + Label::new(["rankings", "level"]).unwrap() +} + +fn unhex(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// The stored JSON the named type writes for `value`. +async fn stored(keyset: &Keyset<'_>, name: &str, value: ValueFrame) -> serde_json::Value { + let bytes = targets::encrypt(name, keyset, &label(), value) + .unwrap() + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +/// The one block ORE term the named type stores for `value`, as bytes. +async fn ob(keyset: &Keyset<'_>, name: &str, value: ValueFrame) -> Vec { + let json = stored(keyset, name, value).await; + let terms = json["ob"].as_array().expect("ob is an array"); + assert_eq!(terms.len(), 1, "{name}: one block ORE term"); + unhex(terms[0].as_str().unwrap()) +} + +/// The terms of `values`, two writes each: the second write of a value is +/// a different ciphertext of the same plaintext. +async fn terms(keyset: &Keyset<'_>, name: &str, values: &[T]) -> Vec> { + let mut out = Vec::new(); + for value in values { + out.push(ob(keyset, name, frame(*value)).await); + out.push(ob(keyset, name, frame(*value)).await); + } + out +} + +async fn text_terms(keyset: &Keyset<'_>, name: &str, values: &[&str]) -> Vec> { + let mut out = Vec::new(); + for value in values { + out.push(ob(keyset, name, frame(*value)).await); + out.push(ob(keyset, name, frame(*value)).await); + } + out +} + +/// ore-rs's own comparator over two terms. +fn rust_cmp(a: &[u8], b: &[u8]) -> Ordering { + let a = OreBlockTerm::::from_bytes(a).unwrap(); + let b = OreBlockTerm::::from_bytes(b).unwrap(); + a.compare(&b).expect("terms of one index compare") +} + +#[tokio::test] +async fn each_type_stores_one_block_ore_term_of_its_scheme() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + for (name, value, header, keys) in [ + ( + "IntegerOrdOre", + frame(7i32), + [2u8, 2], + &["c", "i", "ob", "v"][..], + ), + ( + "SmallintOrdOre", + frame(7i16), + [2, 2], + &["c", "i", "ob", "v"], + ), + ("BigintOrdOre", frame(7i64), [2, 2], &["c", "i", "ob", "v"]), + ("RealOrdOre", frame(7.0f32), [2, 2], &["c", "i", "ob", "v"]), + ( + "DoubleOrdOre", + frame(7.0f64), + [2, 2], + &["c", "i", "ob", "v"], + ), + ( + "TextOrdOre", + frame("alice"), + [2, 3], + &["c", "hm", "i", "ob", "v"], + ), + ( + "TextSearchOre", + frame("alice"), + [2, 3], + &["bf", "c", "hm", "i", "ob", "v"], + ), + ] { + let json = stored(&keyset, name, value).await; + let mut found: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + found.sort_unstable(); + assert_eq!(found, keys, "{name}"); + let term = unhex(json["ob"][0].as_str().unwrap()); + assert_eq!(term[..2], header, "{name}: the scheme's wire header"); + } +} + +/// Pairs whose order keys first differ in each block in turn: `x` with one +/// bit of block `k` clear, and `x` with it set. An `i32`'s key is 32 bits, +/// six 6-bit blocks (the last holds two bits); an `i64`'s is eleven. +fn first_difference_in_every_block_i32() -> Vec { + let mut values = Vec::new(); + for block in 0..6u32 { + let bit = (6 * block + 5).min(31); + let mask = 1u32 << (31 - bit); + let key = 0x5a5a_5a5a_u32 & !mask; + // The order key of an i32 is its bits with the sign bit flipped. + values.push((key ^ 0x8000_0000) as i32); + values.push(((key | mask) ^ 0x8000_0000) as i32); + } + values +} + +fn first_difference_in_every_block_i64() -> Vec { + let mut values = Vec::new(); + for block in 0..11u32 { + let bit = (6 * block + 5).min(63); + let mask = 1u64 << (63 - bit); + let key = 0x5a5a_5a5a_5a5a_5a5a_u64 & !mask; + values.push((key ^ 0x8000_0000_0000_0000) as i64); + values.push(((key | mask) ^ 0x8000_0000_0000_0000) as i64); + } + values +} + +/// Text sharing prefixes, across padding buckets (a string pads to 16, 32 +/// or 64 bytes, its one marker byte counted), and differing first at each +/// position of a 40-character string, so in every block of it. +fn texts() -> Vec { + let mut values: Vec = [ + "", + "a", + "ab", + "abc", + "abd", + "abc ", + "b", + "zz", + // 15 characters (a 16-byte bucket) and 16 (a 32-byte bucket). + "aaaaaaaaaaaaaaa", + "aaaaaaaaaaaaaaaa", + "aaaaaaaaaaaaaaab", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + let base = "mnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz".to_string(); + for at in 0..base.len() { + let mut changed = base.clone().into_bytes(); + changed[at] = b'a' + (changed[at] - b'a' + 1) % 26; + values.push(String::from_utf8(changed).unwrap()); + } + values.push(base); + values +} + +fn connect() -> postgres::Client { + postgres::Client::connect( + &std::env::var("EQL_TEST_DATABASE_URL").expect("EQL_TEST_DATABASE_URL"), + postgres::NoTls, + ) + .unwrap() +} + +/// EQL v4's comparator over two terms. +fn sql_cmp(db: &mut postgres::Client, a: &[u8], b: &[u8]) -> Result { + db.query_one( + "SELECT eql_v4_internal.compare_ore_block_v2_term(\ + ROW($1::bytea)::eql_v4_internal.ore_block_v2_term, \ + ROW($2::bytea)::eql_v4_internal.ore_block_v2_term)", + &[&a, &b], + ) + .map(|row| row.get(0)) +} + +fn as_ordering(n: i32) -> Ordering { + n.cmp(&0) +} + +/// Every pair of `terms`, both ways round: EQL's answer, ore-rs's and the +/// plaintext order all agree. `terms[i]` encrypts `plain[i / 2]`. +fn assert_agree( + db: &mut postgres::Client, + what: &str, + plain: &[P], + terms: &[Vec], +) { + for (i, a) in terms.iter().enumerate() { + for (j, b) in terms.iter().enumerate() { + let (pa, pb) = (&plain[i / 2], &plain[j / 2]); + let expected = pa.partial_cmp(pb).unwrap(); + let sql = as_ordering(sql_cmp(db, a, b).unwrap_or_else(|e| panic!("{what}: {e}"))); + assert_eq!(sql, expected, "{what}: {pa:?} against {pb:?} in SQL"); + assert_eq!( + rust_cmp::(a, b), + expected, + "{what}: {pa:?} against {pb:?} in ore-rs" + ); + } + } +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_compares_fixed_width_terms_as_ore_rs_and_the_plaintexts_do() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + + let mut ints = vec![i32::MIN, -1_000_000, -1, 0, 1, 63, 64, 1_000_000, i32::MAX]; + ints.extend(first_difference_in_every_block_i32()); + ints.sort_unstable(); + ints.dedup(); + let int_terms = runtime.block_on(terms(&keyset, "IntegerOrdOre", &ints)); + assert_agree::(&mut db, "IntegerOrdOre", &ints, &int_terms); + + let smalls = vec![i16::MIN, -300, -1, 0, 1, 63, 64, 300, i16::MAX]; + let small_terms = runtime.block_on(terms(&keyset, "SmallintOrdOre", &smalls)); + assert_agree::(&mut db, "SmallintOrdOre", &smalls, &small_terms); + + let mut bigs = vec![i64::MIN, -1, 0, 1, 1 << 40, i64::MAX]; + bigs.extend(first_difference_in_every_block_i64()); + bigs.sort_unstable(); + bigs.dedup(); + let big_terms = runtime.block_on(terms(&keyset, "BigintOrdOre", &bigs)); + assert_agree::(&mut db, "BigintOrdOre", &bigs, &big_terms); + + // Floats order as float8: -0.0 ties 0.0, and NaN (left out of the + // plaintext comparison, which has no order for it) sorts last, below. + let doubles = [ + f64::NEG_INFINITY, + -1e300, + -1.5, + -0.0, + 0.0, + 1e-300, + 1.0, + 2.0, + 1e300, + f64::INFINITY, + ]; + let double_terms = runtime.block_on(terms(&keyset, "DoubleOrdOre", &doubles)); + assert_agree::(&mut db, "DoubleOrdOre", &doubles, &double_terms); + let nan = runtime.block_on(ob(&keyset, "DoubleOrdOre", frame(f64::NAN))); + for term in &double_terms { + assert_eq!(sql_cmp(&mut db, &nan, term).unwrap(), 1, "NaN sorts last"); + assert_eq!(sql_cmp(&mut db, term, &nan).unwrap(), -1, "NaN sorts last"); + } + assert_eq!(sql_cmp(&mut db, &nan, &nan).unwrap(), 0, "NaN ties NaN"); + + let reals = [ + f32::NEG_INFINITY, + -2.5, + -0.0, + 0.0, + 0.5, + f32::MAX, + f32::INFINITY, + ]; + let real_terms = runtime.block_on(terms(&keyset, "RealOrdOre", &reals)); + assert_agree::(&mut db, "RealOrdOre", &reals, &real_terms); +} + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_compares_chained_text_terms_as_ore_rs_and_the_plaintexts_do() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + // Lower-case ASCII folds to itself, so the fold order is byte order. + let mut values = texts(); + values.sort(); + values.dedup(); + let refs: Vec<&str> = values.iter().map(String::as_str).collect(); + let text_terms = runtime.block_on(text_terms(&keyset, "TextOrdOre", &refs)); + let counts: std::collections::BTreeSet = text_terms + .iter() + .map(|t| u16::from_be_bytes([t[2], t[3]])) + .collect(); + assert!( + counts.len() >= 3, + "the values span padding buckets: {counts:?}" + ); + assert_agree::(&mut db, "TextOrdOre", &values, &text_terms); + + // Text that differs only in case and accents folds to one value. + let a = runtime.block_on(ob(&keyset, "TextOrdOre", frame("Café"))); + let b = runtime.block_on(ob(&keyset, "TextOrdOre", frame("cafe"))); + assert_eq!(sql_cmp(&mut db, &a, &b).unwrap(), 0); +} + +/// ore.rs's vectors for the σ-MMO hash bit +/// (`docs/reviews/2026-06-16-mmo-plpgsql-validation.sql` in ore.rs). +const HASH_VECTORS: [(&str, &str, i32); 12] = [ + ( + "00070e151c232a31383f464d545b6269", + "010e1b2835424f5c697683909daab7c4", + 1, + ), + ( + "1f262d343b424950575e656c737a8188", + "121f2c394653606d7a8794a1aebbc8d5", + 0, + ), + ( + "3e454c535a61686f767d848b9299a0a7", + "23303d4a5764717e8b98a5b2bfccd9e6", + 0, + ), + ( + "5d646b727980878e959ca3aab1b8bfc6", + "34414e5b6875828f9ca9b6c3d0ddeaf7", + 0, + ), + ( + "7c838a91989fa6adb4bbc2c9d0d7dee5", + "45525f6c798693a0adbac7d4e1eefb08", + 0, + ), + ( + "9ba2a9b0b7bec5ccd3dae1e8eff6fd04", + "5663707d8a97a4b1becbd8e5f2ff0c19", + 1, + ), + ( + "bac1c8cfd6dde4ebf2f900070e151c23", + "6774818e9ba8b5c2cfdce9f603101d2a", + 0, + ), + ( + "d9e0e7eef5fc030a11181f262d343b42", + "7885929facb9c6d3e0edfa0714212e3b", + 0, + ), + ( + "f8ff060d141b222930373e454c535a61", + "8996a3b0bdcad7e4f1fe0b1825323f4c", + 0, + ), + ( + "171e252c333a41484f565d646b727980", + "9aa7b4c1cedbe8f5020f1c293643505d", + 0, + ), + ( + "363d444b525960676e757c838a91989f", + "abb8c5d2dfecf90613202d3a4754616e", + 1, + ), + ( + "555c636a71787f868d949ba2a9b0b7be", + "bcc9d6e3f0fd0a1724313e4b5865727f", + 0, + ), +]; + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_hash_bit_matches_ore_rs_vectors() { + let mut db = connect(); + for (f, nonce, expected) in HASH_VECTORS { + let bit: i32 = db + .query_one( + "SELECT eql_v4_internal.ore_block_v2_hash_bit($1::bytea, $2::bytea)", + &[&unhex(f), &unhex(nonce)], + ) + .unwrap() + .get(0); + assert_eq!(bit, expected, "{f} under {nonce}"); + } +} + +fn raises(db: &mut postgres::Client, a: &[u8], b: &[u8], what: &str) -> String { + match sql_cmp(db, a, b) { + Ok(n) => panic!("{what}: compared to {n}, not refused"), + Err(e) => e + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_else(|| e.to_string()), + } +} + +/// Needs an EQL v4 installation. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_refuses_malformed_and_mismatched_terms() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let int = runtime.block_on(ob(&keyset, "IntegerOrdOre", frame(7i32))); + let big = runtime.block_on(ob(&keyset, "BigintOrdOre", frame(7i64))); + let text = runtime.block_on(ob(&keyset, "TextOrdOre", frame("alice"))); + + let mut cases: Vec<(&str, Vec)> = vec![ + ("truncated", int[..int.len() - 1].to_vec()), + ("one byte long", int.iter().copied().chain([0]).collect()), + ("empty", Vec::new()), + ("header alone", int[..4].to_vec()), + ]; + let mut version = int.clone(); + version[0] = 1; + cases.push(("wire version 1", version)); + let mut scheme = int.clone(); + scheme[1] = 4; + cases.push(("an unknown scheme", scheme)); + let mut symbol = int.clone(); + symbol[4] = 64; + cases.push(("a symbol of 64", symbol)); + let mut count = int.clone(); + count[3] += 1; + cases.push(("a block count its length does not have", count)); + let mut policy = text.clone(); + policy[4] = 3; + cases.push(("an unknown padding kind", policy)); + let mut bucket = text.clone(); + bucket[5..7].copy_from_slice(&64u16.to_be_bytes()); + cases.push(("a block count its padding cannot produce", bucket)); + for (what, bad) in &cases { + let message = raises(&mut db, bad, &int, what); + assert!( + message.contains("Malformed ORE block term"), + "{what}: {message}" + ); + let message = raises(&mut db, &text, bad, what); + assert!( + message.contains("Malformed ORE block term"), + "{what}: {message}" + ); + assert!( + !message.contains(&hex(bad)) || bad.is_empty(), + "{what}: the message names the length, never the bytes" + ); + } + + let message = raises(&mut db, &int, &text, "fixed against chained"); + assert!(message.contains("different schemes"), "{message}"); + let message = raises(&mut db, &int, &big, "6 blocks against 11"); + assert!(message.contains("different lengths"), "{message}"); + // The same text term under the no-padding policy, which admits any + // block count: well formed, but padded differently. + let mut unpadded = text.clone(); + unpadded[4..7].copy_from_slice(&[0, 0, 0]); + let message = raises(&mut db, &text, &unpadded, "two padding policies"); + assert!(message.contains("different policies"), "{message}"); + + // A value holds exactly one term. + let error = db + .query_one( + "SELECT eql_v4_internal.compare_ore_block_v2_terms(\ + ARRAY[ROW($1::bytea), ROW($1::bytea)]::eql_v4_internal.ore_block_v2_term[], \ + ARRAY[ROW($1::bytea)]::eql_v4_internal.ore_block_v2_term[])", + &[&int], + ) + .unwrap_err(); + assert!( + error + .as_db_error() + .unwrap() + .message() + .contains("exactly one term"), + "{error}" + ); +} + +/// The longest chained term EQL v4 admits: 1366 blocks, a 1,024-byte string +/// (the longest Stack Encrypt writes, marker byte included). +const MAX_CHAINED_BLOCKS: usize = 1366; + +/// A term of `count` all-zero blocks with the header given, well formed in +/// every respect but the header. +fn zero_term(header: &[u8], count: usize) -> Vec { + let mut term = header.to_vec(); + term.resize(header.len() + 25 * count + 16, 0); + term +} + +fn chained_header(kind: u8, policy: u16, count: usize) -> Vec { + let count = u16::try_from(count).unwrap().to_be_bytes(); + let policy = policy.to_be_bytes(); + vec![2, 3, count[0], count[1], kind, policy[0], policy[1]] +} + +/// Needs an EQL v4 installation. For every padding kind, a spread of +/// policies around each power-of-two bucket and every block count up to +/// past the bound, EQL v4 admits a term exactly when ore-rs does and the +/// term has between one block and [`MAX_CHAINED_BLOCKS`]; for the +/// fixed-width scheme, exactly when ore-rs does. ore-rs parses a chained +/// term of no blocks, which Stack Encrypt never writes (text and bytes go +/// in behind a marker byte), and EQL v4 refuses it. The terms are built in the database, from +/// the header fields alone, so the check does not ship each one. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_admits_the_block_counts_ore_rs_does_up_to_the_bound() { + let mut db = connect(); + db.batch_execute( + "CREATE FUNCTION pg_temp.admits(t bytea) RETURNS boolean LANGUAGE plpgsql AS $$ + BEGIN + PERFORM eql_v4_internal.ore_block_v2_scheme(t); + RETURN true; + EXCEPTION WHEN raise_exception THEN + RETURN false; + END $$", + ) + .unwrap(); + + let mut policies: Vec = vec![0, 1, 2, 3, 5, 7, 49151, 49152, 65535]; + for k in 2..=15 { + let bucket = 1u32 << k; + for p in [bucket - 1, bucket, bucket + 1, bucket * 3 / 4] { + policies.push(u16::try_from(p).unwrap()); + } + } + policies.sort_unstable(); + policies.dedup(); + let mut counts: Vec = (0..=MAX_CHAINED_BLOCKS + 40).collect(); + counts.extend([2048, 2731, 2732, 21846, 43691, 65535]); + + let mut cases: Vec<(u8, u16, usize)> = Vec::new(); + for kind in 0..=3u8 { + let kind_policies: &[u16] = match kind { + 1 | 2 => &policies, + _ => &[0, 1, 16], + }; + for &policy in kind_policies { + for &count in &counts { + cases.push((kind, policy, count)); + } + } + } + + let kinds: Vec = cases.iter().map(|c| i32::from(c.0)).collect(); + let pols: Vec = cases.iter().map(|c| i32::from(c.1)).collect(); + let cnts: Vec = cases.iter().map(|c| c.2 as i32).collect(); + let admitted: Vec = db + .query( + "SELECT pg_temp.admits( + '\\x0203'::bytea || substr(int4send(c), 3, 2) || substr(int4send(k), 4, 1) + || substr(int4send(p), 3, 2) || decode(repeat('00', 25 * c + 16), 'hex')) + FROM unnest($1::int[], $2::int[], $3::int[]) WITH ORDINALITY AS x(k, p, c, n) + ORDER BY n", + &[&kinds, &pols, &cnts], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(admitted.len(), cases.len()); + + let mut both = 0; + for (&(kind, policy, count), &sql) in cases.iter().zip(&admitted) { + let term = zero_term(&chained_header(kind, policy, count), count); + let ore_rs = OreBlockTerm::::from_bytes(&term).is_ok(); + let expected = ore_rs && (1..=MAX_CHAINED_BLOCKS).contains(&count); + assert_eq!( + sql, expected, + "chained kind {kind}, policy {policy}, {count} blocks: ore-rs says {ore_rs}" + ); + both += usize::from(sql); + } + // The grid reaches both answers, and past the bound ore-rs alone admits. + assert!(both > 0 && both < cases.len()); + assert!(cases + .iter() + .zip(&admitted) + .any(|(&(_, _, count), &sql)| count > MAX_CHAINED_BLOCKS + && !sql + && OreBlockTerm::::from_bytes(&zero_term( + &chained_header(0, 0, count), + count + )) + .is_ok())); + + for count in 0..=20usize { + let header = [2, 2, 0, u8::try_from(count).unwrap()]; + let term = zero_term(&header, count); + let sql: bool = db + .query_one("SELECT pg_temp.admits($1)", &[&term]) + .unwrap() + .get(0); + let ore_rs = OreBlockTerm::::from_bytes(&term).is_ok(); + assert_eq!(sql, ore_rs, "fixed-width, {count} blocks"); + } +} + +/// Needs an EQL v4 installation. The longest text Stack Encrypt orders, +/// 1023 bytes, is a term of exactly [`MAX_CHAINED_BLOCKS`] blocks and +/// compares; the same shape one block longer, which ore-rs alone would +/// accept, is refused before any block is compared. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_compares_the_longest_term_and_refuses_one_past_it() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let longest = "z".repeat(1023); + let shorter = format!("{}y", "z".repeat(1022)); + let a = runtime.block_on(ob(&keyset, "TextOrdOre", frame(longest.as_str()))); + let b = runtime.block_on(ob(&keyset, "TextOrdOre", frame(shorter.as_str()))); + assert_eq!( + usize::from(u16::from_be_bytes([a[2], a[3]])), + MAX_CHAINED_BLOCKS + ); + assert_eq!( + as_ordering(sql_cmp(&mut db, &a, &b).unwrap()), + Ordering::Greater + ); + assert_eq!( + as_ordering(sql_cmp(&mut db, &b, &a).unwrap()), + Ordering::Less + ); + assert_eq!( + as_ordering(sql_cmp(&mut db, &a, &a).unwrap()), + Ordering::Equal + ); + + let count = MAX_CHAINED_BLOCKS + 1; + let over = zero_term(&chained_header(0, 0, count), count); + assert!( + OreBlockTerm::::from_bytes(&over).is_ok(), + "ore-rs admits it" + ); + let unpadded = zero_term( + &chained_header(0, 0, MAX_CHAINED_BLOCKS), + MAX_CHAINED_BLOCKS, + ); + for (x, y) in [(&over, &unpadded), (&unpadded, &over)] { + let message = raises(&mut db, x, y, "one block past the bound"); + assert!(message.contains("Malformed ORE block term"), "{message}"); + } +} + +/// Needs an EQL v4 installation. Rows written through the by-name dispatch +/// and queried through a btree index on the order term: `<`, `>=`, `=` and +/// `ORDER BY` find exactly the rows the plaintexts say. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_answers_the_orderings_through_a_btree_index() { + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + db.batch_execute( + "CREATE TEMP TABLE v4_ore_block (id int PRIMARY KEY, \ + level public.eql_v4_integer_ord_ore, title public.eql_v4_text_ord_ore);", + ) + .unwrap(); + let rows: [(i32, i32, &str); 6] = [ + (1, -5, "alice"), + (2, 0, "bob"), + (3, 0, "bob"), + (4, 7, "carol"), + (5, 64, "carola"), + (6, i32::MAX, "zoe"), + ]; + for (id, level, title) in rows { + let (level, title) = runtime.block_on(async { + ( + stored(&keyset, "IntegerOrdOre", frame(level)) + .await + .to_string(), + stored(&keyset, "TextOrdOre", frame(title)) + .await + .to_string(), + ) + }); + db.execute( + "INSERT INTO v4_ore_block VALUES ($1, $2::text::jsonb::public.eql_v4_integer_ord_ore, \ + $3::text::jsonb::public.eql_v4_text_ord_ore)", + &[&id, &level, &title], + ) + .unwrap(); + } + db.batch_execute( + "CREATE INDEX ON v4_ore_block USING btree (eql_v4.ord_term_ore(level)); \ + CREATE INDEX ON v4_ore_block USING btree (eql_v4.ord_term_ore(title)); \ + ANALYZE v4_ore_block; SET enable_seqscan = off;", + ) + .unwrap(); + let probe = |name: &str, value: ValueFrame| -> String { + runtime.block_on(async { + let bytes = targets::query(name, &common::index_of(name), &keyset, &label(), value) + .unwrap() + .await + .unwrap(); + String::from_utf8(bytes).unwrap() + }) + }; + let level = probe("IntegerOrdOre", frame(0i32)); + let title = probe("TextOrdOre", frame("bob")); + for (column, domain, probe) in [ + ("level", "query_integer_ord_ore", &level), + ("title", "query_text_ord_ore", &title), + ] { + let ids = |db: &mut postgres::Client, op: &str| -> Vec { + let sql = format!( + "SELECT id FROM v4_ore_block WHERE {column} {op} $1::text::jsonb::eql_v4.{domain} ORDER BY id" + ); + db.query(&sql, &[probe]) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect() + }; + assert_eq!(ids(&mut db, "<"), [1], "{column} <"); + assert_eq!(ids(&mut db, ">="), [2, 3, 4, 5, 6], "{column} >="); + assert_eq!(ids(&mut db, "="), [2, 3], "{column} ="); + let plan: Vec = db + .query( + &format!( + "EXPLAIN SELECT id FROM v4_ore_block WHERE {column} < $1::text::jsonb::eql_v4.{domain}" + ), + &[probe], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert!( + plan.iter().any(|line| line.contains("Index")), + "{column}: the btree index answers `<`: {plan:#?}" + ); + let ordered: Vec = db + .query( + &format!("SELECT id FROM v4_ore_block ORDER BY eql_v4.ord_term_ore({column}), id"), + &[], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!(ordered, [1, 2, 3, 4, 5, 6], "{column}: ORDER BY"); + } +} diff --git a/packages/eql/tests/encryption/tests/v4_range.rs b/packages/eql/tests/encryption/tests/v4_range.rs index e5038b01d..29eb6e375 100644 --- a/packages/eql/tests/encryption/tests/v4_range.rs +++ b/packages/eql/tests/encryption/tests/v4_range.rs @@ -6,70 +6,75 @@ //! value the single-bound query domain for the same term holds under its term //! key (`ob` for block ORE, `op` for CLLW OPE). //! -//! EQL v4's ordered types are not producible yet: EQL compares block-ORE and -//! CLLW-OPE terms, and Stack Encrypt does not derive those for EQL (#1132, -//! #1236). So the order terms here come from cipherstash-client's ORE and OPE -//! indexers, the code that writes the terms EQL's comparators are built for, -//! run offline over `FakeDataKeySource`'s index key as -//! `ste_vec_selectors.rs` does. The rest of each stored value (`v`, `i`, `c`, -//! and `hm` for text) is a real Stack Encrypt `Integer` or `TextEq`. Nothing -//! here derives an order term itself. +//! The order terms, stored and in each bound, are the engine's: Stack Encrypt +//! derives the block ORE (`ob`) and CLLW OPE (`op`) terms of EQL v4's ordered +//! types, so a row is a real `IntegerOrdOre`, `TextOrdOpe`, …, and a bound is +//! the term the single-bound query of that type carries. Nothing here derives +//! an order term itself. The ordered date, timestamp and numeric types are +//! not producible (the engine derives no order term for those kinds), so the +//! smoke test ranges over the families whose ordered types are. //! //! The PostgreSQL tests need an EQL v4 installation (`mise run //! test:encryption:postgres`); the others need no database. mod common; -use cipherstash_client::encryption::{IndexTerm, OpeIndexer, OreIndexer, Plaintext}; -use cipherstash_client::zerokms; +use eql_bindings::encryption::targets; use eql_bindings::v4::{self, integer, text, DomainType}; -use eql_bindings::Identifier; use serde_json::{json, Value}; -use stack_encrypt::NonEmpty; -use stack_kms::{FakeDataKeySource, IndexKeySource}; - -fn column() -> NonEmpty { - Identifier::for_column("users", "age").unwrap() -} +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::target::IndexSpec; +use stack_encrypt::{Encrypt, KeysetCipher, Label}; fn identifier() -> Value { json!({"t": "users", "c": "age"}) } -/// `FakeDataKeySource`'s index key for the default keyset, as the key type -/// cipherstash-client's indexers take. -async fn index_key() -> zerokms::IndexKey { - let (_, key) = FakeDataKeySource::new().load_index_key(None).await.unwrap(); - zerokms::IndexKey::from(*key.key()) +fn label() -> Label { + Label::new(["users", "age"]).unwrap() } -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() } -/// The `ob` value cipherstash-client writes for `plaintext`: one hex block -/// for a number, one per block for text. `query` asks for the query form. -fn ob(key: &zerokms::IndexKey, plaintext: Plaintext, query: bool) -> Value { - let term = if query { - OreIndexer.encrypt_for_query(&plaintext, key) - } else { - OreIndexer.encrypt(&plaintext, key) - }; - match term.unwrap() { - IndexTerm::OreFull(bytes) => json!([hex(&bytes)]), - IndexTerm::OreArray(blocks) => { - Value::Array(blocks.iter().map(|b| Value::String(hex(b))).collect()) - } - other => panic!("not an ORE term: {other:?}"), - } +/// The value the engine seals for `value` as the EQL type `name`, as the +/// JSON text a driver binds: its order term (`ob` or `op`) is what `between` +/// compares a bound against. +async fn stored( + keyset: &KeysetCipher<'_, common::Observed>, + name: &str, + value: ValueFrame, +) -> String { + let bytes = targets::encrypt(name, keyset, &label(), value) + .unwrap() + .await + .unwrap(); + String::from_utf8(bytes).unwrap() } -/// The `op` value cipherstash-client writes for `plaintext`. -fn op(key: &zerokms::IndexKey, plaintext: Plaintext) -> Value { - match OpeIndexer.encrypt(&plaintext, key).unwrap() { - IndexTerm::OpeFixed(bytes) | IndexTerm::OpeVariable(bytes) => Value::String(hex(&bytes)), - other => panic!("not an OPE term: {other:?}"), - } +/// The order term the engine derives for `value` queried as the EQL type +/// `name` through `index` (`"ore_block"` for a block ORE bound, `"ope"` for a +/// CLLW OPE one): what a range holds under `lo` and `hi`, the same value the +/// single-bound query domain holds under `ob` or `op`. +async fn bound( + keyset: &KeysetCipher<'_, common::Observed>, + name: &str, + index: &str, + value: ValueFrame, +) -> Value { + let spec = IndexSpec::parse(index).unwrap(); + let bytes = targets::query(name, &spec, keyset, &label(), value) + .unwrap() + .await + .unwrap(); + let mut query: Value = serde_json::from_slice(&bytes).unwrap(); + let term = match index { + "ore_block" => "ob", + "ope" => "op", + other => panic!("no range over the {other} index"), + }; + query[term].take() } /// A range over the bounds `lo` and `hi`, read through the binding `T` (so @@ -85,15 +90,18 @@ fn range( #[tokio::test] async fn range_bindings_hold_two_bounds_of_their_term_and_refuse_anything_else() { - let key = index_key().await; - let (lo, hi) = (Plaintext::Int(Some(10)), Plaintext::Int(Some(30))); + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let lo_ob = bound(&keyset, "IntegerOrdOre", "ore_block", frame(10i32)).await; + let hi_ob = bound(&keyset, "IntegerOrdOre", "ore_block", frame(30i32)).await; + let lo_op = bound(&keyset, "IntegerOrd", "ope", frame(10i32)).await; + let hi_op = bound(&keyset, "IntegerOrd", "ope", frame(30i32)).await; // Each bound is what the single-bound query operand holds as `ob`: the // same value reads as either. (A block-ORE term carries a nonce, so two // derivations of one value differ; one derivation is used for both.) - let lo_ob = ob(&key, lo.clone(), true); let ore: integer::IntegerRangeOreQuery = serde_json::from_value(json!({ - "v": 4, "i": identifier(), "lo": lo_ob, "hi": ob(&key, hi.clone(), true), + "v": 4, "i": identifier(), "lo": lo_ob, "hi": hi_ob, })) .unwrap(); let single: integer::IntegerOrdOreQuery = @@ -120,16 +128,16 @@ async fn range_bindings_hold_two_bounds_of_their_term_and_refuse_anything_else() let ope: integer::IntegerRangeOpeQuery = serde_json::from_value(json!({ "v": 4, "i": identifier(), - "lo": op(&key, lo.clone()), "hi": op(&key, hi.clone()), + "lo": lo_op.clone(), "hi": hi_op.clone(), })) .unwrap(); let single: integer::IntegerOrdOpeQuery = serde_json::from_value(json!({ - "v": 4, "i": identifier(), "op": op(&key, lo.clone()), + "v": 4, "i": identifier(), "op": lo_op.clone(), })) .unwrap(); assert_eq!(ope.lo, single.op); - let good = json!({"v": 4, "i": identifier(), "lo": op(&key, lo.clone()), "hi": op(&key, hi)}); + let good = json!({"v": 4, "i": identifier(), "lo": lo_op, "hi": hi_op}); let refused = |change: &dyn Fn(&mut Value)| { let mut value = good.clone(); change(&mut value); @@ -252,13 +260,9 @@ fn sqlstate(error: &postgres::Error) -> String { const INTEGERS: [i32; 12] = [-1000, -5, -1, 0, 1, 7, 10, 20, 29, 30, 31, 1000]; /// Integer rows in all three ordered column domains, each value's real -/// Stack Encrypt `Integer` (`v`, `i`, `c`) with cipherstash-client's order -/// term added: `ob` for `_ord_ore`, `op` for `_ord` and `_ord_ope`. -fn integer_table( - runtime: &tokio::runtime::Runtime, - db: &mut postgres::Client, - key: &zerokms::IndexKey, -) { +/// The engine's `IntegerOrdOre`, `IntegerOrd` and `IntegerOrdOpe` values of +/// each of `INTEGERS`: `ob` for `_ord_ore`, `op` for `_ord` and `_ord_ope`. +fn integer_table(runtime: &tokio::runtime::Runtime, db: &mut postgres::Client) { let (cipher, _) = runtime.block_on(common::cipher()); let keyset = cipher.default_keyset(); db.batch_execute( @@ -271,15 +275,7 @@ fn integer_table( ) .unwrap(); for n in INTEGERS { - let stored: integer::Integer = runtime - .block_on(async { keyset.encrypt_as(&n, column()).await }) - .unwrap(); - let base = serde_json::to_value(&stored).unwrap(); - let with = |k: &str, term: Value| { - let mut v = base.clone(); - v[k] = term; - v.to_string() - }; + let row = |name: &str| runtime.block_on(stored(&keyset, name, frame(n))); db.execute( "INSERT INTO v4_int_range VALUES ($1, $2::text::jsonb::public.eql_v4_integer_ord_ore, @@ -287,9 +283,9 @@ fn integer_table( $4::text::jsonb::public.eql_v4_integer_ord_ope)", &[ &n, - &with("ob", ob(key, Plaintext::Int(Some(n)), false)), - &with("op", op(key, Plaintext::Int(Some(n)))), - &with("op", op(key, Plaintext::Int(Some(n)))), + &row("IntegerOrdOre"), + &row("IntegerOrd"), + &row("IntegerOrdOpe"), ], ) .unwrap(); @@ -302,21 +298,22 @@ fn integer_table( #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_finds_integers_in_an_inclusive_range() { let runtime = runtime(); - let key = runtime.block_on(index_key()); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); { let mut db = connect(); - integer_table(&runtime, &mut db, &key); + integer_table(&runtime, &mut db); let ore = |lo: i32, hi: i32| { range::( - ob(&key, Plaintext::Int(Some(lo)), true), - ob(&key, Plaintext::Int(Some(hi)), true), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(lo))), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(hi))), ) }; let ope = |lo: i32, hi: i32| { range::( - op(&key, Plaintext::Int(Some(lo))), - op(&key, Plaintext::Int(Some(hi))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(lo))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(hi))), ) }; let mut found = |column: &str, domain: &str, range: &str| -> Vec { @@ -405,7 +402,6 @@ fn postgres_between_finds_integers_in_an_inclusive_range() { #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_finds_text_in_an_inclusive_range() { let runtime = runtime(); - let key = runtime.block_on(index_key()); let (cipher, _) = runtime.block_on(common::cipher()); let keyset = cipher.default_keyset(); let words = [ @@ -419,15 +415,9 @@ fn postgres_between_finds_text_in_an_inclusive_range() { ]; let mut rows = Vec::new(); for (id, word) in words.iter().enumerate() { - let stored: text::TextEq = runtime - .block_on(async { keyset.encrypt_as(&word.to_string(), column()).await }) - .unwrap(); - let base = serde_json::to_value(&stored).unwrap(); - let mut ore = base.clone(); - ore["ob"] = ob(&key, Plaintext::Text(Some(word.to_string())), false); - let mut ope = base; - ope["op"] = op(&key, Plaintext::Text(Some(word.to_string()))); - rows.push((id as i32, ore.to_string(), ope.to_string())); + let ore = runtime.block_on(stored(&keyset, "TextOrdOre", frame(word.to_string()))); + let ope = runtime.block_on(stored(&keyset, "TextOrdOpe", frame(word.to_string()))); + rows.push((id as i32, ore, ope)); } { let mut db = connect(); @@ -448,7 +438,6 @@ fn postgres_between_finds_text_in_an_inclusive_range() { ) .unwrap(); } - let text = |s: &str| Plaintext::Text(Some(s.to_owned())); let mut found = |column: &str, domain: &str, range: &str| -> Vec { db.query( &format!( @@ -472,15 +461,28 @@ fn postgres_between_finds_text_in_an_inclusive_range() { ]; for (lo, hi, want) in cases { let ore = range::( - ob(&key, text(lo), true), - ob(&key, text(hi), true), + runtime.block_on(bound( + &keyset, + "TextOrdOre", + "ore_block", + frame(lo.to_string()), + )), + runtime.block_on(bound( + &keyset, + "TextOrdOre", + "ore_block", + frame(hi.to_string()), + )), ); assert_eq!( found("ore", "query_text_range_ore", &ore), want, "ORE [{lo}, {hi}]" ); - let ope = range::(op(&key, text(lo)), op(&key, text(hi))); + let ope = range::( + runtime.block_on(bound(&keyset, "TextOrd", "ope", frame(lo.to_string()))), + runtime.block_on(bound(&keyset, "TextOrd", "ope", frame(hi.to_string()))), + ); assert_eq!( found("ope", "query_text_range_ope", &ope), want, @@ -507,10 +509,11 @@ fn postgres_between_finds_text_in_an_inclusive_range() { #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_uses_the_order_term_index() { let runtime = runtime(); - let key = runtime.block_on(index_key()); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); { let mut db = connect(); - integer_table(&runtime, &mut db, &key); + integer_table(&runtime, &mut db); db.batch_execute( "CREATE INDEX v4_int_range_ore ON v4_int_range USING btree (eql_v4.ord_term_ore(ore)); CREATE INDEX v4_int_range_ope ON v4_int_range USING btree (eql_v4.ord_term(ope)); @@ -519,12 +522,12 @@ fn postgres_between_uses_the_order_term_index() { ) .unwrap(); let ore = range::( - ob(&key, Plaintext::Int(Some(10)), true), - ob(&key, Plaintext::Int(Some(30)), true), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(10i32))), + runtime.block_on(bound(&keyset, "IntegerOrdOre", "ore_block", frame(30i32))), ); let ope = range::( - op(&key, Plaintext::Int(Some(10))), - op(&key, Plaintext::Int(Some(30))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(10i32))), + runtime.block_on(bound(&keyset, "IntegerOrd", "ope", frame(30i32))), ); for (column, domain, index, range) in [ ("ore", "query_integer_range_ore", "v4_int_range_ore", &ore), @@ -597,10 +600,13 @@ fn postgres_between_uses_the_order_term_index() { #[test] #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_range_domains_check_their_shape() { - let key = runtime().block_on(index_key()); - let (lo, hi) = (Plaintext::Int(Some(1)), Plaintext::Int(Some(2))); - let ore = json!({"v": 4, "i": identifier(), "lo": ob(&key, lo.clone(), true), "hi": ob(&key, hi.clone(), true)}); - let ope = json!({"v": 4, "i": identifier(), "lo": op(&key, lo), "hi": op(&key, hi)}); + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let term = + |name: &str, index: &str, n: i32| runtime.block_on(bound(&keyset, name, index, frame(n))); + let ore = json!({"v": 4, "i": identifier(), "lo": term("IntegerOrdOre", "ore_block", 1), "hi": term("IntegerOrdOre", "ore_block", 2)}); + let ope = json!({"v": 4, "i": identifier(), "lo": term("IntegerOrd", "ope", 1), "hi": term("IntegerOrd", "ope", 2)}); { let mut db = connect(); let cast = |db: &mut postgres::Client, value: &Value, domain: &str| { @@ -667,50 +673,34 @@ fn postgres_range_domains_check_their_shape() { } } -/// Five plaintexts of `family`, strictly ascending, as cipherstash-client's -/// indexers take them: what the smoke test stores and ranges over. A new -/// ordered family must add a row here, or the test fails naming it. -fn ascending(family: &str) -> [Plaintext; 5] { - use chrono::{DateTime, NaiveDate}; - use rust_decimal::Decimal; - let date = |y, m, d| Plaintext::NaiveDate(NaiveDate::from_ymd_opt(y, m, d)); - let at = |secs| Plaintext::Timestamp(DateTime::from_timestamp(secs, 0)); - let dec = |n, scale| Plaintext::Decimal(Some(Decimal::new(n, scale))); - let text = |s: &str| Plaintext::Text(Some(s.to_owned())); - match family { - "integer" => [-20, -1, 0, 7, 100].map(|n| Plaintext::Int(Some(n))), - "smallint" => [-300, -1, 0, 5, 300].map(|n| Plaintext::SmallInt(Some(n))), - "bigint" => [i64::MIN + 1, -1, 0, 1 << 40, i64::MAX].map(|n| Plaintext::BigInt(Some(n))), - "date" => [ - date(1969, 12, 31), - date(1970, 1, 1), - date(2000, 2, 29), - date(2024, 6, 1), - date(2099, 12, 31), - ], - "timestamp" => [-86_400, 0, 951_782_400, 1_717_200_000, 4_102_444_800].map(at), - "numeric" => [ - dec(-15, 1), - dec(0, 0), - dec(25, 2), - dec(314_159, 5), - dec(1000, 0), - ], - "text" => ["apple", "banana", "cherry", "damson", "elderberry"].map(text), - // `real` values are exact in f32; both float widths encrypt as f64. - "real" => [-2.5, -0.5, 0.0, 1.25, 1e10].map(|f| Plaintext::Float(Some(f))), - "double" => [-1e300, -1.0, 0.0, 2.5, 1e300].map(|f| Plaintext::Float(Some(f))), +/// Five values of `family`, strictly ascending: what the smoke test stores +/// and ranges over. `None` for a family whose ordered types the engine does +/// not produce. A new ordered family must add a row here, or the test fails +/// naming it. +fn ascending(family: &str) -> Option<[Value; 5]> { + Some(match family { + "integer" => [-20, -1, 0, 7, 100].map(Value::from), + "smallint" => [-300, -1, 0, 5, 300].map(Value::from), + "bigint" => [i64::MIN + 1, -1, 0, 1 << 40, i64::MAX].map(Value::from), + "text" => ["apple", "banana", "cherry", "damson", "elderberry"].map(Value::from), + // `real` values are exact in f32; both float widths order as f64. + "real" => [-2.5, -0.5, 0.0, 1.25, 1e10].map(Value::from), + "double" => [-1e300, -1.0, 0.0, 2.5, 1e300].map(Value::from), + "date" | "timestamp" | "numeric" => return None, other => panic!("no smoke-test values for the ordered family {other:?}: add them here"), - } + }) } -/// The text a smoke-test value seals as for its `c` (and, for text, its -/// `hm` and `bf`): `between` never reads them, and most ordered families -/// have no producible type to seal the value itself. -fn display(plaintext: &Plaintext) -> String { - match plaintext { - Plaintext::Text(Some(s)) => s.clone(), - other => format!("{other:?}"), +/// `value` as the plaintext `family`'s types seal. +fn frame_of(family: &str, value: &Value) -> ValueFrame { + match family { + "integer" => frame(i32::try_from(value.as_i64().unwrap()).unwrap()), + "smallint" => frame(i16::try_from(value.as_i64().unwrap()).unwrap()), + "bigint" => frame(value.as_i64().unwrap()), + "real" => frame(value.as_f64().unwrap() as f32), + "double" => frame(value.as_f64().unwrap()), + "text" => frame(value.as_str().unwrap().to_owned()), + other => panic!("no plaintext for the ordered family {other:?}"), } } @@ -725,24 +715,48 @@ fn display(plaintext: &Plaintext) -> String { #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_between_smoke_test_over_every_ordered_family() { let runtime = runtime(); - let key = runtime.block_on(index_key()); let (cipher, _) = runtime.block_on(common::cipher()); let keyset = cipher.default_keyset(); let mut db = connect(); let ranges = v4::all_range_query(); let mut checked = 0; + let mut skipped = Vec::new(); for family in eql_domains::CATALOG .iter() .filter(|f| !f.range_terms().is_empty()) { - let values = ascending(family.name); let table = format!("v4_smoke_{}", family.name); let columns: Vec<&eql_domains::Domain> = family .domains .iter() .filter(|d| d.terms.iter().any(|t| t.range_suffix().is_some())) .collect(); + // A family's ordered types are producible together or not at all: + // the engine derives no order term for a date, a timestamp or a + // decimal, so there is no row of those to range over. + let producible: Vec = columns + .iter() + .map(|d| { + targets::target(&d.rust_struct_name(family.name)).is_some_and(|t| t.producible) + }) + .collect(); + if producible.iter().all(|p| !p) { + assert!(ascending(family.name).is_none(), "{}", family.name); + skipped.push(family.name); + continue; + } + assert!(producible.iter().all(|p| *p), "{}", family.name); + let values = ascending(family.name) + .unwrap_or_else(|| panic!("{}: its ordered types are producible", family.name)); + let type_of = |suffix: &str| { + family + .domains + .iter() + .find(|d| d.name == suffix) + .unwrap_or_else(|| panic!("{} has no {suffix} domain", family.name)) + .rust_struct_name(family.name) + }; db.batch_execute(&format!( "CREATE TEMP TABLE {table} (id int PRIMARY KEY, {})", columns @@ -754,37 +768,14 @@ fn postgres_between_smoke_test_over_every_ordered_family() { .unwrap(); for (id, value) in values.iter().enumerate() { - let text = display(value); - let (eq, matched) = runtime.block_on(async { - let eq: text::TextEq = keyset.encrypt_as(&text, column()).await.unwrap(); - let matched = match family.name { - "text" => Some( - keyset - .encrypt_as::<_, text::TextMatch>(&text, column()) - .await - .unwrap(), - ), - _ => None, - }; - (eq, matched) - }); - let eq = serde_json::to_value(&eq).unwrap(); let payloads: Vec = columns .iter() .map(|d| { - let mut p = json!({"v": eq["v"], "i": eq["i"], "c": eq["c"]}); - for term in d.terms { - match term { - eql_domains::Term::Hm => p["hm"] = eq["hm"].clone(), - eql_domains::Term::Bloom => { - let m = matched.as_ref().expect("only text has a match term"); - p["bf"] = serde_json::to_value(m).unwrap()["bf"].clone(); - } - eql_domains::Term::Ore => p["ob"] = ob(&key, value.clone(), false), - eql_domains::Term::Ope => p["op"] = op(&key, value.clone()), - } - } - p.to_string() + runtime.block_on(stored( + &keyset, + &d.rust_struct_name(family.name), + frame_of(family.name, value), + )) }) .collect(); let placeholders: Vec = columns @@ -816,13 +807,16 @@ fn postgres_between_smoke_test_over_every_ordered_family() { } for term in family.range_terms() { - let bound = |v: &Plaintext| match term { - eql_domains::Term::Ore => ob(&key, v.clone(), true), - eql_domains::Term::Ope => op(&key, v.clone()), + let (name, index) = match term { + eql_domains::Term::Ore => (type_of("ord_ore"), "ore_block"), + eql_domains::Term::Ope => (type_of("ord"), "ope"), other => unreachable!("{other:?} has no range"), }; + let bound_of = |v: &Value| { + runtime.block_on(bound(&keyset, &name, index, frame_of(family.name, v))) + }; let domain = family.range_query_name(term); - let range = json!({"v": 4, "i": identifier(), "lo": bound(&values[1]), "hi": bound(&values[3])}); + let range = json!({"v": 4, "i": identifier(), "lo": bound_of(&values[1]), "hi": bound_of(&values[3])}); // The binding for the domain accepts the range. ranges .iter() @@ -857,11 +851,13 @@ fn postgres_between_smoke_test_over_every_ordered_family() { if family.name == "text" { let range = |term: eql_domains::Term| { - let bound = |v: &Plaintext| match term { - eql_domains::Term::Ore => ob(&key, v.clone(), true), - _ => op(&key, v.clone()), + let (name, index) = match term { + eql_domains::Term::Ore => ("TextOrdOre", "ore_block"), + _ => ("TextOrd", "ope"), }; - json!({"v": 4, "i": identifier(), "lo": bound(&values[1]), "hi": bound(&values[3])}) + let bound_of = + |v: &Value| runtime.block_on(bound(&keyset, name, index, frame_of("text", v))); + json!({"v": 4, "i": identifier(), "lo": bound_of(&values[1]), "hi": bound_of(&values[3])}) .to_string() }; db.batch_execute(&format!( @@ -908,8 +904,13 @@ fn postgres_between_smoke_test_over_every_ordered_family() { db.batch_execute("RESET enable_seqscan;").unwrap(); } } - // 8 numeric and temporal families with 3 overloads each, and text's 5. - assert_eq!(checked, 29, "every between overload was exercised"); + // The 5 numeric families with 3 overloads each, and text's 5. + assert_eq!( + checked, 20, + "every between overload of a producible type was exercised" + ); + skipped.sort_unstable(); + assert_eq!(skipped, ["date", "numeric", "timestamp"]); } /// Needs an EQL v4 installation. The range parameter must be cast to its @@ -995,7 +996,7 @@ impl Drop for NonSuperuserDatabase { /// Needs an EQL v4 installation, and a superuser on `EQL_TEST_DATABASE_URL` /// who can create a role and a database. A non-superuser install skips the -/// `ore_block_256` btree opclass, and `ore_fallback.sql` disables the +/// block ORE btree opclass, and `ore_fallback.sql` disables the /// single-bound ORE domains. Every ORE range domain is disabled the same way, /// naming its family's OPE range, which stays usable. The bundle is the one /// `mise run test:encryption:postgres` builds and installs. @@ -1017,7 +1018,7 @@ fn postgres_a_non_superuser_install_disables_the_ore_ranges() { "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_opclass c JOIN pg_catalog.pg_am am ON am.oid = c.opcmethod WHERE am.amname = 'btree' AND c.opcdefault - AND c.opcintype = 'eql_v4_internal.ore_block_256'::pg_catalog.regtype)", + AND c.opcintype = 'eql_v4_internal.ore_block_v2'::pg_catalog.regtype)", &[], ) .unwrap() diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 85c005610..0eb777c1c 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -445,14 +445,17 @@ async fn every_producible_type_dispatches_by_name() { } Some(_) => { // Asked through each index the type carries, the query - // carries that index's term, the stored one. TextSearch's - // carries it alone (EQL v4's search query terms are - // optional); every other twin carries all of its terms - // whatever the index. + // carries that index's term: the stored one, save a block + // ORE term, which is encrypted under a fresh nonce each + // time and so only has the stored one's shape. The search + // types' queries carry it alone (EQL v4's search query + // terms are optional); every other twin carries all of its + // terms whatever the index. let term_of = |key: &str| match key { "eq" => "hm", "ope" => "op", "match" => "bf", + "ore_block" => "ob", other => panic!("{name}: no term for index {other}"), }; for key in target.indexes { @@ -468,17 +471,34 @@ async fn every_producible_type_dispatches_by_name() { ); let term = term_of(key); assert!(json[term].is_string() || json[term].is_array(), "{name}"); - assert_eq!( - query[term], json[term], - "{name} via {key}: the query carries the stored term" - ); + if term == "ob" { + let one_hex = |value: &serde_json::Value| { + value + .as_array() + .is_some_and(|a| a.len() == 1 && a[0].is_string()) + }; + assert!( + one_hex(&json[term]) && one_hex(&query[term]), + "{name} via {key}: one block ORE term each" + ); + assert_eq!( + query[term][0].as_str().unwrap().len(), + json[term][0].as_str().unwrap().len(), + "{name} via {key}: the query's term is the stored one's length" + ); + } else { + assert_eq!( + query[term], json[term], + "{name} via {key}: the query carries the stored term" + ); + } let carried: Vec<&str> = target .indexes .iter() .map(|key| term_of(key)) .filter(|term| query.get(term).is_some()) .collect(); - if name == "TextSearch" { + if matches!(name, "TextSearch" | "TextSearchOre") { assert_eq!(carried, [term], "{name} via {key}: its term alone"); } else { assert_eq!(carried.len(), target.indexes.len(), "{name} via {key}"); diff --git a/packages/eql/tests/timing/ore_block_v2/README.md b/packages/eql/tests/timing/ore_block_v2/README.md new file mode 100644 index 000000000..f90bbc6f1 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/README.md @@ -0,0 +1,108 @@ +# EQL v4 block ORE comparator timing harness + +Checks that `eql_v4_internal.compare_ore_block_v2_terms`, the comparator +behind every ordered block ORE operator and operator class in EQL v4, takes +the same time wherever its operands first differ, and whether they differ at +all. EQL v4 compares ore-rs's 6-bit block ORE (the fixed-width scheme and +the chained one), a different format from EQL v3's under the same `ob` key, +with a comparator of its own. + +## Why it matters + +The database operator gains nothing from the comparator's timing: they hold +the stored ciphertexts and can run the comparator on any two of them. The +party who gains is someone who can issue queries and time them without +reading the stored ciphertexts, such as an application user. Query results +already tell them the order of their value relative to stored ones. If the +comparison time depends on where the two values first differ, timing also +tells them how long a prefix they share. + +PL/pgSQL is not a constant-time environment. What it can give is a constant +*operation count*: for operands of one shape, the same statements and +function calls whatever their contents. The shape is the scheme and the two +block counts, which a term's length reveals anyway, so the comparator may +branch on it. The v4 comparator runs one statement per compared block, which +evaluates both of the block's comparisons and latches the first difference +arithmetically, runs the hash exactly once on every path (equal terms +included), and selects its result arithmetically. C-level effects inside +those statements (`memcmp` stopping early within 16 bytes, the offset a +`substr` reads at) remain, orders of magnitude smaller. + +## Running it + +From `packages/eql`: + +```sh +mise run build +# a scratch database with the EQL v4 bundle installed, e.g.: +psql "$DATABASE_URL" -f release/cipherstash-encrypt-v4.sql +DATABASE_URL=postgres://... tests/timing/ore_block_v2/run.sh # 100k samples per kind +DATABASE_URL=postgres://... tests/timing/ore_block_v2/run.sh 20000 # quick check +``` + +It needs `psql` and `cargo`, and no CipherStash credentials: `ctgen/` encrypts +with ore-rs under a random throwaway key. Run it on a quiet machine. It is not +a CI test, because timing on shared runners is too noisy to gate on. + +Everything it creates lives in an `ore_v2_timing` schema, dropped on exit. If +that schema already exists, it refuses to run rather than touch it. + +## Method + +`ctgen` produces real ore-rs 0.9.0 ciphertexts, one term per side, in four +kinds of two classes each, plus a correctness set: + +- **fixed**: `u64` terms of the fixed-width scheme (11 blocks), differing in + the first block against differing only in the last; +- **equal**: a `u64` against a second encryption of the same value (a + different nonce, no block differs), against a first-block difference; +- **chained**: 31-byte text of the chained scheme (a 32-byte bucket, 43 + blocks), differing in the first character against only the last; +- **prefix**: 10-byte text (a 16-byte bucket, 22 blocks) against 26-byte + text (a 32-byte bucket, 43 blocks) whose first 22 blocks equal it, so the + block counts settle the order, against the same shapes differing in the + first block; +- **checks**: 1,500 random pairs of either scheme with shared prefixes of + every length, whose expected sign is the plaintext order and, in `ctgen`, + ore-rs's own comparator's. The harness refuses to time a comparator that + gets any of them wrong, either way round. + +Sampling follows dudect, with three lessons from the ore.rs benches built in: + +1. **Both classes share one table, in random order.** With separate storage + per class, placement alone produced a timing difference. +2. **Several comparisons per sample** (16 for the fixed-width kinds, 8 for + the chained ones), so a sample spans many ticks of `clock_timestamp()`'s + microsecond clock. Each comparison's pair is drawn at random, with + replacement, from that class. +3. **The plain Welch t over every sample**, not a percentile-cropped maximum. + Cropping turns a difference in spread into a large t with no difference in + mean. + +Classes are chosen at random per sample, so drift affects both equally. `|t| < 5` over a hundred thousand samples +counts as no detectable difference. + +## Results + +PostgreSQL 17.5 run natively on macOS, Apple silicon, 100,000 samples per +kind, two runs. The machine was shared with other builds, which shows in +the difference between the runs' means; classes are drawn at random per +sample, so drift affects both equally. + +| kind | run 1 t | run 2 t | mean µs per sample (run 2, class 0 / 1) | +|---|---:|---:|---| +| fixed, first vs last block | −1.22 | +0.33 | 192.3 / 191.4 | +| fixed, equal vs first block | −0.65 | −0.40 | 168.1 / 168.1 | +| chained, first vs last block | +0.74 | +0.09 | 204.5 / 204.5 | +| chained, prefix vs first block | −2.33 | −4.01 | 159.2 / 159.5 | + +None passes |t| = 5. The prefix kind is negative on both runs: class 0, +where the block counts settle the order, is faster by about 0.3 µs per +sample of eight comparisons, some 40 ns a comparison, or 0.2% of one. That +is below the cost of a single statement, so it is a C-level effect of the +kind described above, not a skipped statement. A skipped hash or an early +exit would cost a statement or more per comparison. + +Cost: about 10.5 µs per fixed-width comparison of 11 blocks and about +20 to 25 µs per chained comparison of 22 to 43 blocks, whatever the +operands. diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/.gitignore b/packages/eql/tests/timing/ore_block_v2/ctgen/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/.gitignore @@ -0,0 +1 @@ +/target diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.lock b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.lock new file mode 100644 index 000000000..16d2dfbc8 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.lock @@ -0,0 +1,374 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", + "zeroize", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-block-v2-timing-ctgen" +version = "0.0.0" +dependencies = [ + "hex", + "ore-rs", + "rand", +] + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand", + "rand_chacha", + "subtle-ng", + "thiserror", + "zeroize", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.toml b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.toml new file mode 100644 index 000000000..c9a31f8ff --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/Cargo.toml @@ -0,0 +1,21 @@ +# Ciphertext generator for the EQL v4 block ORE comparator timing harness +# (../run.sh). +# +# A DETACHED crate: the empty `[workspace]` table makes it its own workspace +# root, so it never joins the main workspace or its CI. It encrypts with ore-rs +# directly under a random throwaway key; the harness needs real ciphertexts but +# no CipherStash credentials, and nothing it produces is a test fixture. +[package] +name = "ore-block-v2-timing-ctgen" +version = "0.0.0" +edition = "2021" +publish = false + +[dependencies] +# The 6-bit fixed-width and chained schemes behind EQL v4's `ore_block_v2`; +# the same version as the root Cargo.lock. +ore-rs = { version = "=0.9.0", default-features = false } +rand = "0.8" +hex = "0.4" + +[workspace] diff --git a/packages/eql/tests/timing/ore_block_v2/ctgen/src/main.rs b/packages/eql/tests/timing/ore_block_v2/ctgen/src/main.rs new file mode 100644 index 000000000..203fbef0e --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/ctgen/src/main.rs @@ -0,0 +1,146 @@ +//! ore-rs 6-bit block ORE ciphertexts for the EQL v4 comparator timing +//! harness (`../run.sh`). Tab-separated rows, one term per side, hex: +//! +//! fixed u64, 11 blocks; differ in block 0 / only block 10 +//! equal u64; a second encryption of a / differ in block 0 +//! chained 31-byte text, 43 blocks; differ in the first +//! character / only the last +//! prefix text in a 16-byte bucket against text in a +//! 32-byte bucket: every block a has equals b's +//! (the block counts settle it) / the first differs +//! check random pairs of either scheme with shared +//! prefixes; expected = sign(a cmp b) +//! +//! Every expected sign is the plaintext order, and ore-rs's own comparator +//! is asserted to agree before a row is written. +use std::cmp::Ordering; + +use ore_rs::scheme::bit2_w6::OreAes128Bit6ChaCha20; +use ore_rs::scheme::chained::OreAes128Bit6ChainedChaCha20; +use ore_rs::{OreCipher, OreEncrypt, OreOutput}; +use rand::Rng; + +fn sign(o: Ordering) -> i32 { + match o { + Ordering::Less => -1, + Ordering::Equal => 0, + Ordering::Greater => 1, + } +} + +fn lower(rng: &mut impl Rng, n: usize) -> String { + (0..n).map(|_| rng.gen_range(b'a'..=b'z') as char).collect() +} + +fn main() { + let mut rng = rand::thread_rng(); + let key: [u8; 16] = rng.gen(); + let fixed = OreAes128Bit6ChaCha20::init(&key).unwrap(); + let chained = OreAes128Bit6ChainedChaCha20::init(&key).unwrap(); + let ct = |x: u64| x.encrypt(&fixed).unwrap().to_bytes(); + let st = |s: &str| chained.encrypt_str(s).unwrap().to_bytes(); + let pairs: usize = std::env::args() + .nth(1) + .map(|s| s.parse().unwrap()) + .unwrap_or(256); + let row = |kind: &str, label: &str, a: &[u8], b: &[u8]| { + println!("{kind}\t{label}\t{}\t{}", hex::encode(a), hex::encode(b)); + }; + for _ in 0..pairs { + let x: u64 = rng.gen(); + row("fixed", "first", &ct(x), &ct(x ^ (1 << 63))); + row("fixed", "last", &ct(x), &ct(x ^ 1)); + row("equal", "same", &ct(x), &ct(x)); + row("equal", "first", &ct(x), &ct(x ^ (1 << 63))); + + let s = lower(&mut rng, 31); + let mut first = s.clone().into_bytes(); + first[0] = b'a' + (first[0] - b'a' + 1) % 26; + let mut last = s.clone().into_bytes(); + last[30] = b'a' + (last[30] - b'a' + 1) % 26; + row( + "chained", + "first", + &st(&s), + &st(std::str::from_utf8(&first).unwrap()), + ); + row( + "chained", + "last", + &st(&s), + &st(std::str::from_utf8(&last).unwrap()), + ); + + // 10 bytes pads with NULs to a 16-byte bucket: 22 blocks, the last + // holding two bits of byte 15 and four of byte 16. The longer text + // repeats the padding and puts a byte whose top four bits are clear + // at 16, so all 22 of the shorter text's blocks match it, and its + // 26 bytes take a 32-byte bucket. + let short = lower(&mut rng, 10); + let long = format!("{short}\0\0\0\0\0\0\u{5}{}", lower(&mut rng, 9)); + let (a, b) = (st(&short), st(&long)); + assert_eq!( + OreAes128Bit6ChainedChaCha20::compare_raw_slices(&a, &b), + Some(Ordering::Less) + ); + row("prefix", "prefix", &a, &b); + let mut other = long.clone().into_bytes(); + other[0] = b'a' + (other[0] - b'a' + 1) % 26; + row( + "prefix", + "first", + &a, + &st(std::str::from_utf8(&other).unwrap()), + ); + } + for i in 0..1500 { + if i % 2 == 0 { + let x: u64 = rng.gen(); + let keep = rng.gen_range(0..=64u32); + let mask = if keep == 0 { + 0 + } else { + u64::MAX << (64 - keep) + }; + let y = if i % 6 == 0 { + x + } else { + (x & mask) | (rng.gen::() & !mask) + }; + let (a, b) = (ct(x), ct(y)); + let expected = x.cmp(&y); + assert_eq!( + OreAes128Bit6ChaCha20::compare_raw_slices(&a, &b), + Some(expected) + ); + println!( + "check\t{}\t{}\t{}", + sign(expected), + hex::encode(&a), + hex::encode(&b) + ); + } else { + let len = rng.gen_range(0..=40); + let s = lower(&mut rng, len); + let shared = rng.gen_range(0..=s.len()); + let t = if i % 5 == 0 { + s.clone() + } else { + let tail = rng.gen_range(0..=40 - shared); + format!("{}{}", &s[..shared], lower(&mut rng, tail)) + }; + let (a, b) = (st(&s), st(&t)); + let expected = s.as_bytes().cmp(t.as_bytes()); + assert_eq!( + OreAes128Bit6ChainedChaCha20::compare_raw_slices(&a, &b), + Some(expected) + ); + println!( + "check\t{}\t{}\t{}", + sign(expected), + hex::encode(&a), + hex::encode(&b) + ); + } + } +} diff --git a/packages/eql/tests/timing/ore_block_v2/harness.sql b/packages/eql/tests/timing/ore_block_v2/harness.sql new file mode 100644 index 000000000..9778df9b4 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/harness.sql @@ -0,0 +1,98 @@ +-- EQL v4 block ORE comparator timing harness. Loaded by run.sh into a +-- database with the EQL v4 bundle installed, after run.sh has created the +-- `ore_v2_timing` schema and loaded the ctgen output into +-- ore_v2_timing.staging (`\copy` cannot take a psql variable). Everything +-- lives in `ore_v2_timing`, which run.sh creates and drops afterwards. + +SET search_path = ore_v2_timing, public; + +CREATE FUNCTION term_of(h text) + RETURNS eql_v4_internal.ore_block_v2_term[] + LANGUAGE sql IMMUTABLE AS $$ + SELECT ARRAY[ROW(decode(h, 'hex'))::eql_v4_internal.ore_block_v2_term] +$$; + +CREATE TABLE checks AS + SELECT label::int AS expected, term_of(a_hex) AS a, term_of(b_hex) AS b + FROM staging WHERE kind = 'check'; + +-- Timing pairs. Both classes of a kind share one table in random order, so +-- neither sits in a fixed region of memory: with separate per-class storage, +-- placement alone produced a timing difference in the ore.rs benches. +-- Class 1 is a difference in the first block. Class 0: +-- fixed, chained: a difference in the last block only; +-- equal: a second encryption of the same value (no block differs); +-- prefix: every block of the shorter term matches the longer's, so +-- the block counts settle the order. +CREATE TABLE pairs AS + SELECT kind, row_number() OVER (PARTITION BY kind ORDER BY random()) AS id, + (label = 'first')::int AS cls, term_of(a_hex) AS a, term_of(b_hex) AS b + FROM staging WHERE kind <> 'check'; + +-- dudect-style sampling. Each sample picks a class at random and times +-- `per_sample` comparisons of pairs of that class, each drawn at random with +-- replacement, through compare_ore_block_v2_terms over the composite (the +-- operator-class path). clock_timestamp() ticks in microseconds, so a sample +-- must span many of them. +CREATE FUNCTION bench(which_kind text, samples int, per_sample int) + RETURNS TABLE (cls int, us double precision) +AS $$ +DECLARE + arrs_a eql_v4_internal.ore_block_v2[]; + arrs_b eql_v4_internal.ore_block_v2[]; + idx0 int[]; + idx1 int[]; + picks int[]; + c int; + r int; + t0 timestamptz; + t1 timestamptz; +BEGIN + SELECT array_agg(ROW(p.a)::eql_v4_internal.ore_block_v2 ORDER BY p.id), + array_agg(ROW(p.b)::eql_v4_internal.ore_block_v2 ORDER BY p.id) + INTO arrs_a, arrs_b FROM ore_v2_timing.pairs p WHERE p.kind = which_kind; + SELECT array_agg(p.id::int ORDER BY p.id) FILTER (WHERE p.cls = 0), + array_agg(p.id::int ORDER BY p.id) FILTER (WHERE p.cls = 1) + INTO idx0, idx1 FROM ore_v2_timing.pairs p WHERE p.kind = which_kind; + FOR s IN 1..samples LOOP + c := (random() < 0.5)::int; + IF c = 0 THEN + picks := ARRAY(SELECT idx0[1 + floor(random() * cardinality(idx0))::int] FROM generate_series(1, per_sample)); + ELSE + picks := ARRAY(SELECT idx1[1 + floor(random() * cardinality(idx1))::int] FROM generate_series(1, per_sample)); + END IF; + t0 := clock_timestamp(); + FOREACH r IN ARRAY picks LOOP + PERFORM eql_v4_internal.compare_ore_block_v2_terms(arrs_a[r], arrs_b[r]); + END LOOP; + t1 := clock_timestamp(); + cls := c; + us := extract(epoch FROM t1 - t0) * 1e6; + RETURN NEXT; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +-- Uncropped Welch t (every sample counts), with per-class mean and median. +-- Report this, not a percentile-cropped maximum: cropping turns a difference +-- in spread into a large t with no difference in mean. +CREATE FUNCTION welch(label text, tbl regclass) + RETURNS TABLE (run text, n bigint, mean_0 numeric, mean_1 numeric, + median_0 numeric, median_1 numeric, t numeric) +AS $$ +BEGIN + RETURN QUERY EXECUTE format($q$ + WITH s AS ( + SELECT count(*) n, + count(*) FILTER (WHERE cls = 0) n0, count(*) FILTER (WHERE cls = 1) n1, + avg(us) FILTER (WHERE cls = 0) m0, avg(us) FILTER (WHERE cls = 1) m1, + stddev(us) FILTER (WHERE cls = 0) s0, stddev(us) FILTER (WHERE cls = 1) s1, + percentile_cont(0.5) WITHIN GROUP (ORDER BY us) FILTER (WHERE cls = 0) p0, + percentile_cont(0.5) WITHIN GROUP (ORDER BY us) FILTER (WHERE cls = 1) p1 + FROM %s) + SELECT %L::text, n, round(m0::numeric, 1), round(m1::numeric, 1), + round(p0::numeric, 1), round(p1::numeric, 1), + round(((m0 - m1) / sqrt(s0 * s0 / n0 + s1 * s1 / n1))::numeric, 2) + FROM s$q$, tbl, label); +END; +$$ LANGUAGE plpgsql; diff --git a/packages/eql/tests/timing/ore_block_v2/run.sh b/packages/eql/tests/timing/ore_block_v2/run.sh new file mode 100755 index 000000000..159d0ce24 --- /dev/null +++ b/packages/eql/tests/timing/ore_block_v2/run.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Timing harness for EQL v4's block ORE comparator (compare_ore_block_v2_terms). +# By hand, not CI: timing on shared runners is too noisy to gate on. +# +# DATABASE_URL=postgres://... tests/timing/ore_block_v2/run.sh [samples] +# +# DATABASE_URL must point at a database with the EQL v4 bundle installed (e.g. +# release/cipherstash-encrypt-v4.sql from `mise run build`). Needs psql and +# cargo. samples defaults to 100000 per kind, a few minutes in all. See +# README.md for the method and how to read the output. +set -euo pipefail + +samples="${1:-100000}" +here="$(cd "$(dirname "$0")" && pwd)" +: "${DATABASE_URL:?set DATABASE_URL to a database with the EQL v4 bundle installed}" + +# Everything lives in the ore_v2_timing schema, which is dropped on exit. +# Refuse to start if it already exists, so a run never drops a schema it did +# not make. A failed psql is reported as itself, not as an existing schema. +if ! exists="$(psql "$DATABASE_URL" -tA -c "SELECT to_regnamespace('ore_v2_timing') IS NOT NULL")"; then + echo "could not query the database at DATABASE_URL (see the psql error above)" >&2 + exit 1 +fi +if [ "$exists" != "f" ]; then + echo "schema ore_v2_timing already exists; not touching it" >&2 + exit 1 +fi + +tmp="$(mktemp -d)" +created=0 +cleanup() { + rm -rf "$tmp" + if [ "$created" = 1 ]; then + psql "$DATABASE_URL" -q -c "DROP SCHEMA ore_v2_timing CASCADE" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +echo "== generating ciphertexts (ore-rs, throwaway key)" +cargo run --locked --quiet --release --manifest-path "$here/ctgen/Cargo.toml" -- 256 >"$tmp/ct.tsv" + +psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 -c "CREATE SCHEMA ore_v2_timing" >/dev/null +created=1 +psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 \ + -c "CREATE TABLE ore_v2_timing.staging (kind text, label text, a_hex text, b_hex text)" \ + -c "\\copy ore_v2_timing.staging FROM '$tmp/ct.tsv'" >/dev/null +psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 -f "$here/harness.sql" >/dev/null + +echo "== correctness (must be 0 wrong before timing means anything)" +wrong="$(psql "$DATABASE_URL" -tA -c " + SELECT count(*) FILTER (WHERE eql_v4_internal.compare_ore_block_v2_terms(a, b) IS DISTINCT FROM expected) + + count(*) FILTER (WHERE eql_v4_internal.compare_ore_block_v2_terms(b, a) IS DISTINCT FROM -expected) + FROM ore_v2_timing.checks")" +echo " wrong: $wrong" +[ "$wrong" = "0" ] || { echo "comparator disagrees with plaintext order; not timing"; exit 1; } + +echo "== timing (two runs per kind; classes: see README.md)" +for run in 1 2; do + psql "$DATABASE_URL" -q -v ON_ERROR_STOP=1 -c " + CREATE TABLE ore_v2_timing.fixed_$run AS SELECT * FROM ore_v2_timing.bench('fixed', $samples, 16); + CREATE TABLE ore_v2_timing.equal_$run AS SELECT * FROM ore_v2_timing.bench('equal', $samples, 16); + CREATE TABLE ore_v2_timing.chained_$run AS SELECT * FROM ore_v2_timing.bench('chained', $samples, 8); + CREATE TABLE ore_v2_timing.prefix_$run AS SELECT * FROM ore_v2_timing.bench('prefix', $samples, 8);" +done +psql "$DATABASE_URL" -c " + SELECT * FROM ore_v2_timing.welch('fixed, first vs last block, 16/sample #1', 'ore_v2_timing.fixed_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('fixed, first vs last block, 16/sample #2', 'ore_v2_timing.fixed_2') + UNION ALL SELECT * FROM ore_v2_timing.welch('fixed, equal vs first block, 16/sample #1', 'ore_v2_timing.equal_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('fixed, equal vs first block, 16/sample #2', 'ore_v2_timing.equal_2') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, first vs last block, 8/sample #1', 'ore_v2_timing.chained_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, first vs last block, 8/sample #2', 'ore_v2_timing.chained_2') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, prefix vs first block, 8/sample #1', 'ore_v2_timing.prefix_1') + UNION ALL SELECT * FROM ore_v2_timing.welch('chained, prefix vs first block, 8/sample #2', 'ore_v2_timing.prefix_2');" diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 6e0649ac3..76a369a06 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -9,6 +9,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A block ORE index, `OreBlock`** (ADR-0002 and ADR-0009, amended + 2026-10-19). It derives ore-rs 0.9.0's 6-bit block ORE under its own key + domain, over the same order keys as the CLLW indexes: + `ore_rs::vitaminc::Bit6BlockOre` for targets of at most ten bytes (the + integers from 16 to 64 bits and both floats) and `ChainedBlockOre` for + text, bytes and the 128-bit integers, text padded to a power-of-two bucket + of at least 16 bytes. As for ORE and OPE, a boolean and an 8-bit integer + have no block ORE term. `sem::OreBlockTerm` is the term, the whole ciphertext; it carries + a random nonce, so it compares by the scheme's comparator (`compare`, + `PartialEq`, `PartialOrd`), never by its bytes, and + `OreBlockTerm::from_bytes` checks a stored term's structure. The typed + path reaches it with `.via(OreBlock)` and `target::ore_block()`, the + dynamic path as the index `ore_block`, and `KeysetCipher::ore_block_term` + derives one directly. A query is the full ciphertext too: EQL v4's + comparator reads the right half of the operand a btree index passes + second. Text whose fold is longer than 1023 bytes is refused with + `TermError::OreBlockTooLong`, text whose fold ends in NUL with + `TermError::OreBlockTrailingNul`, before any key is touched on the query + path; `TermError::OreBlock` carries a scheme failure and + `TermBytesError::MalformedOreBlockCiphertext` a malformed stored term. The + CLLW `Ore` and `Ope` indexes are unchanged, byte for byte. + - **Every integer width, 128-bit integers, dates, timestamps and decimals as dynamic values** (`dynamic` feature; #1144, ADR-0002). They are vitaminc leaves in a `ValueFrame`, with no new `FfiValue` variant: @@ -196,6 +218,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking +- **`IndexSpec` has a fifth variant, `OreBlock`** (the block ORE index + above). The enum is exhaustive on purpose, since a binding has to be + taught each index, so a `match` over it needs an arm for `OreBlock`. The + dynamic path answers an ordering through ORE, else block ORE, else OPE, + whichever the field declares. - **A target's query is handed the index that answers its operator** (`dynamic` feature). `TargetResolver::query` and `record::query` take an `&IndexSpec` after the type's name and the field respectively; diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 858816166..2cfc02e0a 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -50,6 +50,9 @@ serde = { workspace = true } base64ct = { version = "1.7", features = ["alloc"] } cllw-ore = { workspace = true } +# Block ORE (`OreBlock`): ore-rs's 6-bit schemes through the same +# vitaminc-ore traits cllw-ore implements. +ore-rs = { workspace = true } # Every error derives `miette::Diagnostic` (a code, and help where a caller # can act) and gives its structured fields as a `serde_json` map through # `ErrorPayload`. Both are already in the graph through stack-kms. diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index d083210bf..fe99b1ab5 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -1,6 +1,7 @@ --- status: proposed date: 2026-10-13 +amended: 2026-10-19 supersedes: ADR-0004 (decision 5) extends: ADR-0003, ADR-0007 --- @@ -41,6 +42,7 @@ Decision 5 of ADR-0004 assumed they were. In general they are not: | --- | --- | --- | | HMAC | the HMAC | the same HMAC | | ORE | left and right ciphertext | the left ciphertext only | +| block ORE (6-bit; amended 2026-10-19) | the whole ore-rs ciphertext, left and right | the whole ciphertext too, under a fresh nonce: EQL v4's comparator reads its first operand's left half and its second operand's right half, and a btree index or `between` may pass the stored value first. Equal values compare equal; the bytes never match | | OPE | the OPE value of the order key | the same value; equal terms mean equal order keys, which is plaintext equality only when the key is lossless (it is for integers, and not for text after the column's declared transform) | | Bloom match | one filter over every n-gram of the value | a filter over the n-grams of the needle: a subset, or bits the stored filter lacks | | Bloom prefix (planned) | an HMAC per edge n-gram | one HMAC | @@ -510,6 +512,19 @@ text order encoding, the system ADR-0002's fold pinned to Unicode 16, and the ORE and OPE indexes answer the orderings on text over it. `TextOrdOre` and `TextSearchOre` also need the block ORE scheme EQL stores (#1132). +*Amended 2026-10-19:* they have it. A new index, `IndexSpec::OreBlock` +(wire key `ore_block`), derives ore-rs's 6-bit block ORE under its own key +domain: `Bit6BlockOre` for targets of at most ten bytes (the integers to 64 +bits and the floats) and `ChainedBlockOre` for text, bytes and the 128-bit +integers, over the same order keys the CLLW indexes use. It answers the four +orderings exactly. Its query term is the full ciphertext (see the table +above), and text it cannot take is refused before any term exists: text +whose fold is longer than 1023 bytes or ends in NUL. The CLLW `Ore` and +`Ope` indexes are unchanged, byte for byte. EQL's `ob` key is this index +(`eql-codegen`'s `index_key` maps it to `ore_block`, not to `ore`, which is +CLLW ORE and has no EQL domain), so `TextOrdOre`, `TextSearchOre` and the +integer and float `*OrdOre` types are produced. + A `Ranked` term's `target_limit` applies the index's configured recall factor and needs no `Stats`. diff --git a/packages/stack-encrypt/fuzz/Cargo.lock b/packages/stack-encrypt/fuzz/Cargo.lock index e1111571c..ac5eabd82 100644 --- a/packages/stack-encrypt/fuzz/Cargo.lock +++ b/packages/stack-encrypt/fuzz/Cargo.lock @@ -21,6 +21,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -227,6 +228,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "bumpalo" version = "3.19.1" @@ -239,6 +246,12 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -1355,6 +1368,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1365,6 +1392,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" @@ -1380,6 +1416,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1411,6 +1468,35 @@ dependencies = [ "libc", ] +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.6", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1995,6 +2081,7 @@ dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", "stack-encrypt-derive", @@ -2092,6 +2179,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.119" diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index 3a03cc178..5cf19530a 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -79,6 +79,9 @@ mod tests { errors.extend(boxed(variants![ TermError::Prf(_) => TermError::Prf(cause()), TermError::Ore(_) => TermError::Ore(cllw_ore::Error), + TermError::OreBlockTooLong => TermError::OreBlockTooLong, + TermError::OreBlockTrailingNul => TermError::OreBlockTrailingNul, + TermError::OreBlock(_) => TermError::OreBlock(cause()), TermError::InvalidOptions(_) => TermError::InvalidOptions("k out of range"), TermError::EmptyTermText => TermError::EmptyTermText, TermError::NoEqualityTerm => TermError::NoEqualityTerm, @@ -101,6 +104,9 @@ mod tests { TermBytesError::MalformedCllwCiphertext(_) => { TermBytesError::MalformedCllwCiphertext(3) }, + TermBytesError::MalformedOreBlockCiphertext(_) => { + TermBytesError::MalformedOreBlockCiphertext(3) + }, ])); errors.extend(boxed(variants![ LabelError::Empty => LabelError::Empty, @@ -405,6 +411,12 @@ mod tests { Box::new(TermBytesError::MalformedCllwCiphertext(5)), json!({ "len": 5 }), ), + ( + Box::new(TermBytesError::MalformedOreBlockCiphertext(7)), + json!({ "len": 7 }), + ), + (Box::new(TermError::OreBlockTooLong), json!({})), + (Box::new(TermError::OreBlockTrailingNul), json!({})), ( Box::new(TermBytesError::MatchPositionOutOfRange { position: 900, diff --git a/packages/stack-encrypt/src/dynamic/kind.rs b/packages/stack-encrypt/src/dynamic/kind.rs index d127f98b0..039d47b7e 100644 --- a/packages/stack-encrypt/src/dynamic/kind.rs +++ b/packages/stack-encrypt/src/dynamic/kind.rs @@ -55,10 +55,10 @@ use crate::target::IndexSpec; /// `vitaminc-normalize` gives an equality domain, which is every integer /// of every width, both floats, a decimal, a date, a timestamp, text and /// bytes, and not a boolean. The rest are this crate's: match over text -/// alone, and ORE and OPE over the integers of 16 bits and wider, both -/// floats, text and bytes. A boolean and an 8-bit integer have no order -/// term: over so small a domain it would reveal which rows are `true`, or -/// each row's rank among 256 values. +/// alone, and ORE, OPE and block ORE over the integers of 16 bits and +/// wider, both floats, text and bytes. A boolean and an 8-bit integer have +/// no order term: over so small a domain it would reveal which rows are +/// `true`, or each row's rank among 256 values. /// A decimal, a date and a timestamp have none yet: their order encodings /// wait for new cllw-ore and ore-rs releases (#1236), and [`not_admitted`] /// names that reason. A composite (`array`, `object`) has no term. Only the @@ -86,7 +86,7 @@ pub fn admits(kind: ValueKind, index: &IndexSpec) -> bool { /// Why `kind` does not admit `index`, or `None` when it does: the reason a /// plan that declares the pair is refused with. /// -/// [`Reason::OrderTermPending`] for an ORE or OPE index on a decimal, date +/// [`Reason::OrderTermPending`] for an order index on a decimal, date /// or timestamp, whose order terms wait for new cllw-ore and ore-rs /// releases (#1236). [`Reason::IndexNotAdmitted`] for every other pair /// [`admits`] refuses. @@ -103,7 +103,7 @@ pub fn not_admitted(kind: ValueKind, index: &IndexSpec) -> Option { if admits(kind, index) { return None; } - let ordering = matches!(index, IndexSpec::Ore | IndexSpec::Ope); + let ordering = matches!(index, IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock); let pending = matches!( kind, ValueKind::Decimal | ValueKind::Date | ValueKind::Timestamp @@ -581,12 +581,13 @@ mod tests { } } - fn indexes() -> [IndexSpec; 4] { + fn indexes() -> [IndexSpec; 5] { [ IndexSpec::Equality, IndexSpec::Match(MatchOptions::default()), IndexSpec::Ore, IndexSpec::Ope, + IndexSpec::OreBlock, ] } @@ -622,6 +623,9 @@ mod tests { assert!(!admits(ValueKind::Bool, &IndexSpec::Ope)); assert!(!admits(ValueKind::Object, &IndexSpec::Ope)); assert!(!admits(ValueKind::Array, &IndexSpec::Ore)); + assert!(admits(ValueKind::Int32, &IndexSpec::OreBlock)); + assert!(admits(ValueKind::String, &IndexSpec::OreBlock)); + assert!(!admits(ValueKind::Bool, &IndexSpec::OreBlock)); } /// Equality is vitaminc's table: a kind admits it exactly when @@ -677,6 +681,7 @@ mod tests { assert!(admits(kind, &IndexSpec::Equality), "{kind} equality"); assert!(admits(kind, &IndexSpec::Ore), "{kind} ore"); assert!(admits(kind, &IndexSpec::Ope), "{kind} ope"); + assert!(admits(kind, &IndexSpec::OreBlock), "{kind} ore_block"); assert!(!admits(kind, &matched), "{kind} match"); } // An 8-bit integer keeps its equality term and, like a boolean, @@ -696,6 +701,7 @@ mod tests { assert!(admits(kind, &IndexSpec::Equality), "{kind} equality"); assert!(!admits(kind, &IndexSpec::Ore), "{kind} ore"); assert!(!admits(kind, &IndexSpec::Ope), "{kind} ope"); + assert!(!admits(kind, &IndexSpec::OreBlock), "{kind} ore_block"); assert!(!admits(kind, &matched), "{kind} match"); } } @@ -709,11 +715,12 @@ mod tests { for index in indexes() { let reason = not_admitted(kind, &index); assert_eq!(reason.is_none(), admits(kind, &index), "{kind} and {index}"); - let pending = matches!(index, IndexSpec::Ore | IndexSpec::Ope) - && matches!( - kind, - ValueKind::Decimal | ValueKind::Date | ValueKind::Timestamp - ); + let pending = + matches!(index, IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock) + && matches!( + kind, + ValueKind::Decimal | ValueKind::Date | ValueKind::Timestamp + ); if pending { assert_eq!(reason, Some(Reason::OrderTermPending), "{kind} and {index}"); } else if reason.is_some() { @@ -737,11 +744,18 @@ mod tests { not_admitted(ValueKind::Date, &IndexSpec::Match(MatchOptions::default())), Some(Reason::IndexNotAdmitted) ); - assert_eq!(not_admitted(ValueKind::Int16, &IndexSpec::Ore), None); assert_eq!( - not_admitted(ValueKind::Int8, &IndexSpec::Ore), - Some(Reason::IndexNotAdmitted) + not_admitted(ValueKind::Timestamp, &IndexSpec::OreBlock), + Some(Reason::OrderTermPending) ); + assert_eq!(not_admitted(ValueKind::Int16, &IndexSpec::Ore), None); + for index in [IndexSpec::Ore, IndexSpec::Ope, IndexSpec::OreBlock] { + assert_eq!( + not_admitted(ValueKind::Int8, &index), + Some(Reason::IndexNotAdmitted), + "an 8-bit integer is refused {index} as a boolean is, not as pending" + ); + } assert!(Reason::OrderTermPending.to_string().contains("#1236")); } diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index fe1726039..22ee6f526 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -36,15 +36,15 @@ //! declared kind: `eq` through an equality index, `matches` through a match //! index over text, and the orderings (`lt`, `lte`, `gt`, `gte`, //! `between`) through an ORE index over an integer of 16 bits or wider, a -//! float or text, or an OPE index when the field declares no ORE. The typed -//! path's default for an ordering is ORE, and OPE is the typed path's -//! `.via(Ope)`; here a field that declares only OPE is answered through it. -//! Every order term is derived over the value's ADR-0002 order key: a -//! float's ties `-0.0` with `0.0` and sorts every NaN last, and text's is -//! its fold. An ordering over bytes is not answered, as `op::Ordered` does -//! not cover them. Nor is one over an 8-bit integer, which has no order -//! term, or over a decimal, date or timestamp: a plan cannot declare an -//! order index on one yet. +//! float or text, else a block ORE index, else an OPE index. The typed +//! path's default for an ordering is ORE, and block ORE and OPE are the +//! typed path's `.via(OreBlock)` and `.via(Ope)`; here a field that +//! declares only one of them is answered through it. Every order term is +//! derived over the value's ADR-0002 order key: a float's ties `-0.0` with +//! `0.0` and sorts every NaN last, and text's is its fold. An ordering over +//! bytes is not answered, as `op::Ordered` does not cover them. Nor is one +//! over an 8-bit integer, which has no order term, or over a decimal, date +//! or timestamp: a plan cannot declare an order index on one yet. //! //! A field that names an EQL type answers what its //! [`TargetDescriptor::operators`] lists: the operators of a single value @@ -189,6 +189,7 @@ impl OpSpec { return None; } declared(|index| matches!(index, IndexSpec::Ore)) + .or_else(|| declared(|index| matches!(index, IndexSpec::OreBlock))) .or_else(|| declared(|index| matches!(index, IndexSpec::Ope))) } } @@ -612,6 +613,15 @@ fn check_input(index: &IndexSpec, input: &Input) -> Result<(), Error> { order_text(lo).map_err(refused)?; order_text(hi).map_err(refused)?; } + // Block ORE refuses more order text: a fold longer than 1023 + // bytes, or one that ends in NUL. + (IndexSpec::OreBlock, Input::One(value)) => { + ore_block_text(value).map_err(refused)?; + } + (IndexSpec::OreBlock, Input::Pair(lo, hi)) => { + ore_block_text(lo).map_err(refused)?; + ore_block_text(hi).map_err(refused)?; + } (IndexSpec::Match(options), Input::One(value)) => { query_tokens(value, options).map(drop)?; } @@ -643,6 +653,22 @@ fn order_text(value: &ValueFrame) -> Result<(), TermError> { .unwrap_or(Ok(())) } +/// Whether `value`, if it is order text, has a block ORE term: its fold +/// exists, is at most 1023 bytes and does not end in NUL +/// ([`crate::sem::ore_block_text`]). Anything but text has nothing to +/// refuse here. +fn ore_block_text(value: &ValueFrame) -> Result<(), TermError> { + value + .visit(OneScalar::new( + |ty: Option, value: Physical<'_>| match (ty, value) { + (None, Physical::Text(text)) => Ok(crate::sem::ore_block_text(text)), + _ => Err(None), + }, + )) + // Not one text scalar: nothing to refuse here. + .unwrap_or(Ok(())) +} + /// The tokens a match query of `value` keeps, under the index's options. /// /// # Errors @@ -1483,6 +1509,59 @@ mod tests { assert_eq!(negative_zero.operand, zero.operand); } + /// Block ORE takes order text whose fold is at most 1023 bytes and does + /// not end in NUL. A query is refused for any other, either bound of a + /// range included, before any term exists, by the key-free check too. + #[tokio::test] + async fn block_ore_refuses_text_it_has_no_term_for_before_any_term() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = record::plan(&frame_of(&Tree::Object(vec![field( + "title", + "string", + vec![text("c"), text("ore_block")], + )]))) + .expect("the plan parses"); + let one = |text: &str| ValueFrame::of(text).expect("a value"); + let two = |lo: &str, hi: &str| ValueFrame::of(vec![lo, hi]).expect("a value"); + let longest = "a".repeat(1023); + let too_long = "a".repeat(1024); + let refused = [ + (OpSpec::Lt, one(&too_long), "long"), + (OpSpec::Gt, one("a\0"), "nul"), + (OpSpec::Between, two("a", &too_long), "long"), + (OpSpec::Between, two(&too_long, "b"), "long"), + (OpSpec::Between, two("a\0", "b"), "nul"), + (OpSpec::Between, two("a", "b\0"), "nul"), + ]; + for (op, input, what) in refused { + let derived = match query_term(&keyset, &plan, "title", op, &input, &NoTargets) { + Ok(pending) => pending.await.map_err(Error::Cipher).map(|_| ()), + Err(error) => Err(error), + }; + for error in [ + derived.unwrap_err(), + check_query(&plan, "title", op, &input, &NoTargets).unwrap_err(), + ] { + let term = term_error(&error); + let expected = match what { + "long" => matches!(term, Some(TermError::OreBlockTooLong)), + _ => matches!(term, Some(TermError::OreBlockTrailingNul)), + }; + assert!(expected, "{op:?} {what}: {error:?}"); + } + } + for input in [one(&longest), one("a\0b"), one("")] { + check_query(&plan, "title", OpSpec::Lt, &input, &NoTargets) + .expect("block ORE takes it"); + let term = query_term(&keyset, &plan, "title", OpSpec::Lt, &input, &NoTargets) + .expect("a query") + .await + .expect("a term"); + assert_eq!(term.index, IndexSpec::OreBlock); + } + } + /// A plan whose context is a field of each record has no context a /// query could derive under; the binding sends the plan under the /// context it names. diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index e39a91b35..576c9b806 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -1218,6 +1218,8 @@ enum OutputWire { Match(Option), Ore, Ope, + #[serde(rename = "ore_block")] + OreBlock, } impl From for Output { @@ -1229,6 +1231,7 @@ impl From for Output { OutputWire::Match(options) => IndexWire::Match(options), OutputWire::Ore => IndexWire::Ore, OutputWire::Ope => IndexWire::Ope, + OutputWire::OreBlock => IndexWire::OreBlock, }; Output::Term(index.into()) } diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index 1bcb28b7b..48f360461 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -15,12 +15,13 @@ //! compared in Unicode 16 NFC, and a date, timestamp, decimal or uuid has //! a domain of its own. A boolean has none. //! - **Match** takes text only, cut in Unicode 16 NFC. -//! - **ORE and OPE** take an integer of 16 bits or wider, a float, text or -//! bytes, each handed to cllw-ore as the Rust type the typed path would -//! have named: an `i16` as an `i16`, a `u128` as a `u128`. A boolean, a -//! `u8` and an `i8` have no order term: over a domain that small (two -//! values, or 256) it would tell anyone holding the column which rows are -//! `true`, or rank every row among 256. A decimal, a date and a timestamp +//! - **ORE, OPE and block ORE** take an integer of 16 bits or wider, a +//! float, text or bytes, each handed to cllw-ore (ORE, OPE) or ore-rs +//! (block ORE) as the Rust type the typed path would have named: an `i16` +//! as an `i16`, a `u128` as a `u128`. A boolean, a `u8` and an `i8` have +//! no order term: over a domain that small (two values, or 256) it would +//! tell anyone holding the column which rows are `true`, or rank every +//! row among 256. A decimal, a date and a timestamp //! have none yet: cllw-ore 0.5 does not encode them the ADR-0002 way, and //! their order terms wait for new cllw-ore and ore-rs releases (#1236). //! @@ -53,7 +54,8 @@ use crate::{Encrypt, IntoPrfContext, KeysetCipher, NonEmpty}; /// /// # Wire form /// -/// An index is its key, a string: `"eq"`, `"match"`, `"ore"` or `"ope"`. +/// An index is its key, a string: `"eq"`, `"match"`, `"ore"`, `"ope"` or +/// `"ore_block"`. /// `"match"` is a match index under the default options /// ([`MatchOptions::default`]), so every plan written before options had a /// wire form reads as it always did. @@ -69,7 +71,7 @@ use crate::{Encrypt, IntoPrfContext, KeysetCipher, NonEmpty}; /// Every option is optional and defaults to [`MatchOptions::default`]'s /// value; an unknown option, an option given twice, or options that fail /// the match scheme's bounds (`k` in `3..=16`, `m` a power of two in -/// `[32, 65536]`, a non-zero n-gram length) are refused. The other three +/// `[32, 65536]`, a non-zero n-gram length) are refused. The other four /// indexes have no options and no object form. /// /// Serialized (it is `Serialize`), an index writes the string whenever the @@ -85,6 +87,7 @@ impl IndexSpec { "match" => IndexSpec::Match(MatchOptions::default()), "ore" => IndexSpec::Ore, "ope" => IndexSpec::Ope, + "ore_block" => IndexSpec::OreBlock, _ => return None, }) } @@ -109,9 +112,10 @@ impl IndexSpec { /// is one scalar the index takes. Equality takes every scalar vitaminc /// gives an equality domain (an integer, a float, text, bytes, and a /// date, timestamp, decimal or uuid), not a boolean or a null. Match - /// takes text. ORE and OPE take an integer of any width, a float, text - /// or bytes; not a boolean, and not yet a decimal, date or timestamp - /// (#1236). A match index's options do not change what it takes. + /// takes text. ORE, OPE and block ORE take an integer of any width, a + /// float, text or bytes; not a boolean, and not yet a decimal, date or + /// timestamp (#1236). A match index's options do not change what it + /// takes. /// /// This is the one table: [`term`] refuses exactly what it refuses, and /// checking it first is how a binding refuses a bad request at its @@ -148,18 +152,18 @@ impl IndexSpec { // vitaminc's table: a kind with an equality domain. IndexSpec::Equality => domain_of(ty, value).is_some(), IndexSpec::Match(_) => ty.is_none() && matches!(value, Physical::Text(_)), - IndexSpec::Ore | IndexSpec::Ope => ty.is_none() && ordered(value), + IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock => ty.is_none() && ordered(value), } } } -/// Whether CLLW ORE and OPE order this scalar: an integer of 16 bits or -/// wider, a float, text or bytes. Not a boolean, and not an 8-bit integer: -/// an order term over so small a domain reveals too much, which rows hold -/// `true`, or each row's rank among 256 values, which a few known rows -/// resolve to the values. Not a null. The caller has already refused a -/// scalar under a logical type: a decimal, date or timestamp's order term -/// waits for a later release. +/// Whether the order indexes (CLLW ORE and OPE, and block ORE) order this +/// scalar: an integer of 16 bits or wider, a float, text or bytes. Not a +/// boolean, and not an 8-bit integer: an order term over so small a domain +/// reveals too much, which rows hold `true`, or each row's rank among 256 +/// values, which a few known rows resolve to the values. Not a null. The +/// caller has already refused a scalar under a logical type: a decimal, +/// date or timestamp's order term waits for a later release. fn ordered(value: &Physical<'_>) -> bool { match value { Physical::Int(Int::I8(_) | Int::U8(_)) | Physical::Null | Physical::Bool(_) => false, @@ -181,6 +185,8 @@ impl Serialize for IndexSpec { Match(MatchOut), Ore, Ope, + #[serde(rename = "ore_block")] + OreBlock, } #[derive(Serialize)] struct MatchOut { @@ -209,6 +215,7 @@ impl Serialize for IndexSpec { }), IndexSpec::Ore => Out::Ore, IndexSpec::Ope => Out::Ope, + IndexSpec::OreBlock => Out::OreBlock, } .serialize(s) } @@ -225,6 +232,8 @@ pub(super) enum IndexWire { Match(Option), Ore, Ope, + #[serde(rename = "ore_block")] + OreBlock, } impl From for IndexSpec { @@ -234,6 +243,7 @@ impl From for IndexSpec { IndexWire::Match(options) => IndexSpec::Match(options.map(|o| o.0).unwrap_or_default()), IndexWire::Ore => IndexSpec::Ore, IndexWire::Ope => IndexSpec::Ope, + IndexWire::OreBlock => IndexSpec::OreBlock, } } } @@ -478,12 +488,21 @@ where .match_terms_under::(text, context, options.clone()) .map(|terms| TermBytes(terms.to_bytes()))) } - IndexSpec::Ore => ore_of(cipher, value, context).ok_or(None), - IndexSpec::Ope => ope_of(cipher, value, context).ok_or(None), + IndexSpec::Ore => order_of(cipher, OrderIndex::Ore, value, context).ok_or(None), + IndexSpec::Ope => order_of(cipher, OrderIndex::Ope, value, context).ok_or(None), + IndexSpec::OreBlock => order_of(cipher, OrderIndex::OreBlock, value, context).ok_or(None), } } -/// [`IndexSpec::Ore`] per scalar, each handed to the CLLW encryptor as the +/// The order index a scalar's term is derived under. +#[derive(Clone, Copy)] +enum OrderIndex { + Ore, + Ope, + OreBlock, +} + +/// An order index's term per scalar, each handed to the encryptor as the /// Rust type the typed path would have named; `None` for a scalar it does /// not order ([`ordered`]). /// @@ -497,8 +516,9 @@ where /// Each scalar is encoded at the target of the Rust type named here /// ([`Orderable`]), as on the typed path: text by the system ADR-0002's /// fold, so `café` precomposed or with a combining accent, and `Café`, tie. -fn ore_of<'a, 'c, K: 'static, D>( +fn order_of<'a, 'c, K: 'static, D>( cipher: &'a KeysetCipher<'_, K>, + index: OrderIndex, value: Physical<'_>, context: NonEmpty, ) -> Option> @@ -506,51 +526,27 @@ where D: IntoPrfContext<'c>, { Some(match value { - Physical::Int(Int::I16(v)) => ore(cipher, v, context), - Physical::Int(Int::I32(v)) => ore(cipher, v, context), - Physical::Int(Int::I64(v)) => ore(cipher, v, context), - Physical::Int(Int::I128(v)) => ore(cipher, v, context), - Physical::Int(Int::U16(v)) => ore(cipher, v, context), - Physical::Int(Int::U32(v)) => ore(cipher, v, context), - Physical::Int(Int::U64(v)) => ore(cipher, v, context), - Physical::Int(Int::U128(v)) => ore(cipher, v, context), - Physical::Float(Float::F32(v)) => ore(cipher, v, context), - Physical::Float(Float::F64(v)) => ore(cipher, v, context), - Physical::Text(t) => ore(cipher, t.to_owned(), context), - Physical::Bytes(b) => ore(cipher, b.to_vec(), context), + Physical::Int(Int::I16(v)) => order(cipher, index, v, context), + Physical::Int(Int::I32(v)) => order(cipher, index, v, context), + Physical::Int(Int::I64(v)) => order(cipher, index, v, context), + Physical::Int(Int::I128(v)) => order(cipher, index, v, context), + Physical::Int(Int::U16(v)) => order(cipher, index, v, context), + Physical::Int(Int::U32(v)) => order(cipher, index, v, context), + Physical::Int(Int::U64(v)) => order(cipher, index, v, context), + Physical::Int(Int::U128(v)) => order(cipher, index, v, context), + Physical::Float(Float::F32(v)) => order(cipher, index, v, context), + Physical::Float(Float::F64(v)) => order(cipher, index, v, context), + Physical::Text(t) => order(cipher, index, t.to_owned(), context), + Physical::Bytes(b) => order(cipher, index, b.to_vec(), context), _ => return None, }) } -/// [`IndexSpec::Ope`] per scalar; see [`ore_of`]. -fn ope_of<'a, 'c, K: 'static, D>( - cipher: &'a KeysetCipher<'_, K>, - value: Physical<'_>, - context: NonEmpty, -) -> Option> -where - D: IntoPrfContext<'c>, -{ - Some(match value { - Physical::Int(Int::I16(v)) => ope(cipher, v, context), - Physical::Int(Int::I32(v)) => ope(cipher, v, context), - Physical::Int(Int::I64(v)) => ope(cipher, v, context), - Physical::Int(Int::I128(v)) => ope(cipher, v, context), - Physical::Int(Int::U16(v)) => ope(cipher, v, context), - Physical::Int(Int::U32(v)) => ope(cipher, v, context), - Physical::Int(Int::U64(v)) => ope(cipher, v, context), - Physical::Int(Int::U128(v)) => ope(cipher, v, context), - Physical::Float(Float::F32(v)) => ope(cipher, v, context), - Physical::Float(Float::F64(v)) => ope(cipher, v, context), - Physical::Text(t) => ope(cipher, t.to_owned(), context), - Physical::Bytes(b) => ope(cipher, b.to_vec(), context), - _ => return None, - }) -} - -/// The term's raw bytes are its frozen encoding. -fn ore<'a, 'c, K: 'static, T, D>( +/// The term of `value` under `index`. The term's bytes are its frozen +/// encoding: the raw CLLW ciphertext, or the serialised block ORE one. +fn order<'a, 'c, K: 'static, T, D>( cipher: &'a KeysetCipher<'_, K>, + index: OrderIndex, value: T, context: NonEmpty, ) -> Pending<'a, TermBytes, K> @@ -558,24 +554,17 @@ where T: Orderable, D: IntoPrfContext<'c>, { - cipher - .ore_term(value, context) - .map(|term| TermBytes(term.as_ref().to_vec())) -} - -/// See [`ore`]. -fn ope<'a, 'c, K: 'static, T, D>( - cipher: &'a KeysetCipher<'_, K>, - value: T, - context: NonEmpty, -) -> Pending<'a, TermBytes, K> -where - T: Orderable, - D: IntoPrfContext<'c>, -{ - cipher - .ope_term(value, context) - .map(|term| TermBytes(term.as_ref().to_vec())) + match index { + OrderIndex::Ore => cipher + .ore_term(value, context) + .map(|term| TermBytes(term.as_ref().to_vec())), + OrderIndex::Ope => cipher + .ope_term(value, context) + .map(|term| TermBytes(term.as_ref().to_vec())), + OrderIndex::OreBlock => cipher + .ore_block_term(value, context) + .map(|term| TermBytes(term.into_bytes())), + } } /// A sink that takes one scalar and hands it to `F`, and refuses anything @@ -1472,6 +1461,7 @@ mod tests { default_match(), IndexSpec::Ore, IndexSpec::Ope, + IndexSpec::OreBlock, ] { let wire = crate::dynamic::test_frame::value_of(&spec); assert!( diff --git a/packages/stack-encrypt/src/lib.rs b/packages/stack-encrypt/src/lib.rs index 8afe073e5..8d523019a 100644 --- a/packages/stack-encrypt/src/lib.rs +++ b/packages/stack-encrypt/src/lib.rs @@ -301,7 +301,7 @@ pub use stack_kms::{diagnostic, ErrorPayload}; pub use target::{ CallerContext, CipherScope, DecryptField, DecryptFrom, DecryptInto, Decryptable, Decryption, EncryptFrom, EncryptInto, Encrypted, Encryption, Equality, Index, IndexSpec, Indexes, Match, - Ope, Ore, Pending, PendingFuture, Request, Responses, TermSet, + Ope, Ore, OreBlock, Pending, PendingFuture, Request, Responses, TermSet, }; // Re-export the vitaminc AEAD surface callers need to drive the cipher, so they diff --git a/packages/stack-encrypt/src/op/mod.rs b/packages/stack-encrypt/src/op/mod.rs index de362f76c..8e849f80a 100644 --- a/packages/stack-encrypt/src/op/mod.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -86,10 +86,10 @@ pub(crate) mod stats; pub use stats::{Limit, Stats, DEFAULT_CONFIDENCE, MAX_ROWS}; use crate::sem::{ - DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, OpeTerm, Orderable, OreTerm, - QueryTokens, RangeQuery, + DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, OpeTerm, Orderable, + OreBlockTerm, OreTerm, QueryTokens, RangeQuery, }; -use crate::target::{DeclaredContext, Equality, Index, Match, Ope, Ore, Owned, Pending}; +use crate::target::{DeclaredContext, Equality, Index, Match, Ope, Ore, OreBlock, Owned, Pending}; use crate::{Error, KeysetCipher, Label, NonEmpty, PlanError}; /// A query operator: a unit type naming what a query asks of the server, @@ -515,6 +515,7 @@ impl QueryTerm for EqualityTerm {} impl QueryTerm for MatchQuery {} impl QueryTerm for OreTerm {} impl QueryTerm for OpeTerm {} +impl QueryTerm for OreBlockTerm {} impl QueryTerm for RangeQuery {} /// A query term whose answer can hold **false positives and no false @@ -753,11 +754,14 @@ where } } -/// The ordering operators on the ORE and OPE indexes, over the integer -/// types ([`Ordered`]). The query term for a comparison is the index's own -/// term for the value: this crate's ORE is CLLW, whose ciphertext has no -/// left half to send alone, so what is sent is what is stored, as for an -/// HMAC. `Between` sends both bounds' terms as one `RangeQuery`. +/// The ordering operators on the ORE, OPE and block ORE indexes, over the +/// [`Ordered`] types. The query term for a comparison is the index's own +/// term for the value. CLLW ORE's ciphertext has no left half to send +/// alone, so what is sent is what is stored, as for an HMAC. Block ORE's +/// does, but EQL v4's comparator reads the right half of the operand a +/// btree index or `between` passes second, which is the query, so a block +/// ORE query is the full ciphertext too. `Between` sends both bounds' +/// terms as one `RangeQuery`. macro_rules! orders { ($($index:ident => $term:ident),+ $(,)?) => {$( orders!(@one $index => $term; Lt, Lte, Gt, Gte); @@ -793,10 +797,10 @@ macro_rules! orders { } )+}; } -orders!(Ore => OreTerm, Ope => OpeTerm); +orders!(Ore => OreTerm, Ope => OpeTerm, OreBlock => OreBlockTerm); -/// The ordering operators default to the ORE index; an OPE index is named -/// with `.via(Ope)`. +/// The ordering operators default to the ORE index; an OPE or block ORE +/// index is named with `.via(Ope)` or `.via(OreBlock)`. macro_rules! default_ore { ($($op:ident),+ $(,)?) => {$( impl DefaultIndex for $op { diff --git a/packages/stack-encrypt/src/plan/error.rs b/packages/stack-encrypt/src/plan/error.rs index 2e558b374..1faf306b0 100644 --- a/packages/stack-encrypt/src/plan/error.rs +++ b/packages/stack-encrypt/src/plan/error.rs @@ -316,7 +316,9 @@ fn index_value(index: &IndexSpec) -> serde_json::Value { } }) } - IndexSpec::Equality | IndexSpec::Ore | IndexSpec::Ope => index.key().into(), + IndexSpec::Equality | IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock => { + index.key().into() + } } } diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 36da3b018..8910dcda2 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -182,6 +182,12 @@ const ORE_KEY_DOMAIN: &[u8] = b"stack-encrypt/sem/ore-key/v2"; /// encrypt-only, so the two schemes must never share a key). `v2` as for /// [`ORE_KEY_DOMAIN`]. const OPE_KEY_DOMAIN: &[u8] = b"stack-encrypt/sem/ope-key/v2"; +/// PAE domain for block ORE key derivation, distinct from both CLLW +/// domains so no two schemes share a key. One key per index: ore-rs derives +/// the 6-bit scheme's key and the chained scheme's from it under labels of +/// its own, and binds the scheme, the target and the context into each +/// term's salt. +const ORE_BLOCK_KEY_DOMAIN: &[u8] = b"stack-encrypt/sem/ore-block-key/v1"; /// Errors from SEM term generation. #[derive(Debug, thiserror::Error, miette::Diagnostic)] @@ -199,6 +205,35 @@ pub enum TermError { #[error("ORE/OPE encryption failed: {0}")] #[diagnostic(code(stack_encrypt::ore_failed))] Ore(#[from] cllw_ore::Error), + /// Block ORE ([`OreBlockTerm`]) orders text and bytes of at most 1023 + /// bytes, counted after the order encoding (text folded): the chained + /// scheme's 1024-byte ceiling, less the one marker byte it puts in + /// front of a variable-width value. The ceiling bounds what one value + /// costs: a term of 1023 bytes is already about 34 KB. + #[error( + "the value is too long for a block ORE term: its order encoding is longer than 1023 bytes" + )] + #[diagnostic( + code(stack_encrypt::ore_block_too_long), + help("Block ORE orders text and bytes of at most 1023 bytes, counted after text is folded. Order a shorter prefix of the value, or index the column with OPE.") + )] + OreBlockTooLong, + /// Text whose order encoding ends in NUL has no block ORE term: the + /// chained scheme pads text with NUL bytes to its length bucket, so + /// `"a\0"` would encrypt the same as `"a"`. PostgreSQL `text` cannot + /// hold NUL. + #[error("text that ends in NUL has no block ORE term")] + #[diagnostic( + code(stack_encrypt::ore_block_trailing_nul), + help("Block ORE pads text with NUL bytes, so text that ends in NUL would order as the text without it. Remove the trailing NUL characters before encrypting.") + )] + OreBlockTrailingNul, + /// Block ORE encryption failed in the scheme: its key or its + /// randomness. The scheme's error is the + /// [`source`](std::error::Error::source); it names no part of the value. + #[error("block ORE encryption failed")] + #[diagnostic(code(stack_encrypt::ore_block_failed))] + OreBlock(#[source] Box), /// The supplied [`MatchOptions`] are invalid. #[error("invalid match options: {0}")] #[diagnostic(code(stack_encrypt::invalid_match_options))] @@ -252,6 +287,9 @@ impl crate::ErrorPayload for TermError { Self::Bytes(error) => error.payload(), Self::Prf(_) | Self::Ore(_) + | Self::OreBlockTooLong + | Self::OreBlockTrailingNul + | Self::OreBlock(_) | Self::InvalidOptions(_) | Self::EmptyTermText | Self::NoEqualityTerm @@ -308,6 +346,12 @@ pub enum TermBytesError { #[error("{0} bytes do not fit this CLLW ciphertext shape")] #[diagnostic(code(stack_encrypt::cllw_ciphertext_length))] MalformedCllwCiphertext(usize), + /// The bytes are not a block ORE ciphertext ore-rs's comparator reads: + /// neither the 6-bit scheme's header and length nor the chained + /// scheme's. The length is all there is to report. + #[error("{0} bytes are not a block ORE ciphertext")] + #[diagnostic(code(stack_encrypt::ore_block_ciphertext_shape))] + MalformedOreBlockCiphertext(usize), } impl crate::ErrorPayload for TermBytesError { @@ -316,7 +360,8 @@ impl crate::ErrorPayload for TermBytesError { match self { Self::WrongEqualityTermLength(len) | Self::OddMatchTermsLength(len) - | Self::MalformedCllwCiphertext(len) => payload([("len", (*len).into())]), + | Self::MalformedCllwCiphertext(len) + | Self::MalformedOreBlockCiphertext(len) => payload([("len", (*len).into())]), Self::MatchPositionOutOfRange { filter_size, .. } => { payload([("filter_size", (*filter_size).into())]) } @@ -1233,6 +1278,7 @@ index_term!(EqualityTerm); index_term!(MatchTerms, O: MatchConfig); index_term!(OreTerm, T: Orderable); index_term!(OpeTerm, T: Orderable); +index_term!(OreBlockTerm, T: Orderable); macro_rules! term_wrapper { ($name:ident, $inner:ty) => { @@ -1334,6 +1380,156 @@ macro_rules! term_wrapper { term_wrapper!(OreTerm, OreCllw8VariableV1); term_wrapper!(OpeTerm, OpeCllw8VariableV1); +/// A block ORE index term for a source value of type `T`: ore-rs's 6-bit +/// block ORE encryption of the value's `vitaminc-ore` order encoding at +/// `T`'s [`Orderable`] target, the scheme EQL v4's `_ord_ore` domains +/// store (`ob`). Comparisons order like the plaintexts, within one index. +/// +/// Which of ore-rs's two 6-bit schemes encrypts it is fixed by the target: +/// +/// - **`Bit6BlockOre`** for a target of at most ten bytes: the integers to +/// 64 bits and floats (as the `f64` they widen to). Wire header +/// `02 02`, then the block count; one block per six bits of the +/// encoding, so an `i32` is six blocks and an `i64` eleven. +/// - **`ChainedBlockOre`** for text, bytes and the 128-bit integers. Wire +/// header `02 03`, the block count, then the padding policy. Text is +/// padded with NUL bytes to a power of two of at least 16 bytes, so a +/// term reveals its text's length bucket rather than its length; bytes +/// and integers are not padded. Text and bytes of more than 1023 bytes +/// are refused ([`TermError::OreBlockTooLong`]), and text that ends in +/// NUL ([`TermError::OreBlockTrailingNul`]). +/// +/// A term is the whole ciphertext, left and right halves. The right half +/// carries a random nonce, so one value encrypts to different bytes each +/// time, and those terms compare equal. That is why equality here is the +/// scheme's comparison, not the bytes: `==` is "orders equal", and there +/// is no `Eq` or `Hash`, since two terms of different indexes, or a +/// malformed one, have no order at all. +/// +/// The query term for every ordering operator is this same full +/// ciphertext, not its left half alone: EQL v4's comparator takes the +/// stored term's left half and the query's right half when a btree index +/// or `eql_v4.between` passes the stored value first. +/// +/// Comparing two stored terms needs no key, as for any ORE: a full +/// ciphertext carries its deterministic left half, so anyone holding two +/// stored terms learns their order and, for the chained scheme, the length +/// of their common prefix in six-bit blocks. +pub struct OreBlockTerm(Vec, PhantomData T>); + +/// Which of ore-rs's 6-bit schemes encrypts a target's terms: the +/// fixed-width scheme for a target of at most ten bytes, the chained one +/// for the rest, text and bytes among them. +fn ore_block_is_bit6(target: OrderTarget) -> bool { + target.width().is_some_and(|width| width <= 10) +} + +/// The order of two block ORE ciphertexts, by the comparator of the scheme +/// their header names; `None` if either is not a well-formed ciphertext of +/// that scheme, if they name different schemes, or (chained) if they were +/// padded under different policies. +/// +/// These are the comparators `Bit6BlockOre` and `ChainedBlockOre` compare +/// with. ore-rs's `bit2_w6` module is its 6-bit scheme; the legacy 8-bit +/// scheme (`BlockOre`, module `bit2`) is not used here. +fn ore_block_compare(a: &[u8], b: &[u8]) -> Option { + use ore_rs::scheme::bit2_w6::OreAes128Bit6ChaCha20; + use ore_rs::scheme::chained::OreAes128Bit6ChainedChaCha20; + use ore_rs::OreCipher; + match (a.get(..2)?, b.get(..2)?) { + ([0x02, 0x02], [0x02, 0x02]) => OreAes128Bit6ChaCha20::compare_raw_slices(a, b), + ([0x02, 0x03], [0x02, 0x03]) => OreAes128Bit6ChainedChaCha20::compare_raw_slices(a, b), + _ => None, + } +} + +impl OreBlockTerm { + /// Decode a term from its frozen byte encoding, the serialised ore-rs + /// ciphertext, for terms persisted server-side. Structural only: the + /// bytes must be a well-formed 6-bit or chained block ORE ciphertext + /// (header, block count, length, padding policy and symbols); the + /// caller asserts they were generated for this source type `T` and + /// under the same context. + pub fn from_bytes(bytes: &[u8]) -> Result { + if ore_block_compare(bytes, bytes).is_none() { + return Err(TermBytesError::MalformedOreBlockCiphertext(bytes.len())); + } + Ok(Self(bytes.to_vec(), PhantomData)) + } + + /// The frozen byte encoding: the serialised ore-rs ciphertext, header + /// first, unframed. EQL v4 stores it hex-encoded as the one element of + /// `ob`. + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + /// Owned copy of [`as_bytes`](Self::as_bytes). + pub fn to_bytes(&self) -> Vec { + self.0.clone() + } + + /// Unwrap into the serialised ciphertext. + pub fn into_bytes(self) -> Vec { + self.0 + } + + /// The order of the two values, by the scheme's comparator. `None` + /// for two terms made under different padding policies, which a + /// decoded term can be; two terms of one index always compare. Terms + /// of different indexes compare as noise: keep an index's terms + /// together. + pub fn compare(&self, other: &Self) -> Option { + ore_block_compare(&self.0, &other.0) + } +} + +/// The inherent `from_bytes` as a std conversion — the same decoder. +impl TryFrom<&[u8]> for OreBlockTerm { + type Error = TermBytesError; + + fn try_from(bytes: &[u8]) -> Result { + Self::from_bytes(bytes) + } +} + +/// The frozen byte encoding — the same bytes as the inherent `as_bytes`. +impl AsRef<[u8]> for OreBlockTerm { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +/// The length only: a term is not secret, but it is long. +impl fmt::Debug for OreBlockTerm { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OreBlockTerm") + .field("len", &self.0.len()) + .finish() + } +} + +impl Clone for OreBlockTerm { + fn clone(&self) -> Self { + Self(self.0.clone(), PhantomData) + } +} + +/// Two terms are equal when their values order equal. Their bytes +/// differ: each carries a random nonce. +impl PartialEq for OreBlockTerm { + fn eq(&self, other: &Self) -> bool { + self.compare(other) == Some(std::cmp::Ordering::Equal) + } +} + +/// The scheme's comparison ([`OreBlockTerm::compare`]). +impl PartialOrd for OreBlockTerm { + fn partial_cmp(&self, other: &Self) -> Option { + self.compare(other) + } +} + /// A [`PrfVisitor`] that carries the plaintext in and hands the CLLW ORE /// ciphertext out. The PRF block keys cllw-ore's vitaminc ORE scheme /// *inside* `visit_block` and dies there: the block arrives by value, is @@ -1397,6 +1593,55 @@ where } } +/// The block ORE twin of [`OreVisitor`]: same key handling, under its own +/// domain, and the scheme chosen by the target ([`ore_block_is_bit6`]). +/// Both schemes are keyed from the one block; ore-rs derives each one's +/// ORE key from it under a label of its own. The plaintext is held in a +/// [`Zeroizing`], as [`OreVisitor`]'s is: ore-rs reads it by reference and +/// works on scrubbed copies, so that drop is the last place its bytes live. +struct OreBlockVisitor(Zeroizing); + +impl PrfVisitor<[u8; 32], P> for OreBlockVisitor +where + T: Orderable, + P: Send + 'static, +{ + type Value = Result, TermError>; + + fn visit_block(self, mut block: [u8; 32]) -> Result { + use ore_rs::vitaminc::{Bit6BlockOre, ChainedBlockOre}; + let key = Protected::new(block); + block.zeroize(); + let term = if ore_block_is_bit6(T::TARGET) { + let scheme = Bit6BlockOre::new(key); + sealed::Sealed::encrypt_in(&*self.0, &scheme.column(T::TARGET, ())) + .map(|term| term.as_ref().to_vec()) + } else { + let scheme = ChainedBlockOre::new(key); + sealed::Sealed::encrypt_in(&*self.0, &scheme.column(T::TARGET, ())) + .map(|term| term.as_ref().to_vec()) + }; + Ok(term.map_err(ore_block_error)) + } +} + +/// Why ore-rs produced no block ORE term, as a [`TermError`]: the value's +/// encoding refused it, the value is too long, text ends in NUL, or the +/// scheme failed. ore-rs's errors name no part of the value. A target the +/// scheme does not take cannot occur ([`ore_block_is_bit6`] sends every +/// other target to the chained scheme, which takes them all), and reads as +/// a scheme failure rather than a panic. +fn ore_block_error(error: OrderError) -> TermError { + use ore_rs::OreError; + match error { + OrderError::Term(error) => TermError::Normalize(error), + OrderError::Backend(OreError::TooLong) => TermError::OreBlockTooLong, + OrderError::Backend(OreError::TrailingNul) => TermError::OreBlockTrailingNul, + OrderError::Backend(error) => TermError::OreBlock(Box::new(error)), + other => TermError::OreBlock(other.to_string().into()), + } +} + /// Derive an ORE term: PRF of the encoded context under a PAE-encoded /// `[domain, context]` framing — the same framing every other term kind uses /// (see the module docs), so no reimplementation of this derivation can @@ -1444,6 +1689,48 @@ fn ope( .map(OpeTerm::new) } +/// Derive a block ORE term — as [`ore`], under the block ORE domain, so +/// no CLLW scheme shares its key. +fn ore_block( + prf: &HmacSha256Prf, + value: T, + context: Context<'_>, +) -> Result, TermError> { + // Wrapped before anything can fail, so a refused value is wiped too. + let value = Zeroizing::new(value); + let context_bytes = context.as_bytes(); + context_bytes + .prf_visit_with_context( + prf, + Context::pae(&[ORE_BLOCK_KEY_DOMAIN, context_bytes]), + OreBlockVisitor(value), + ) + .into_result() + .map_err(TermError::from_prf)? + .map(|bytes| OreBlockTerm(bytes, PhantomData)) +} + +/// Whether block ORE takes `text` as order text: its fold (the order +/// encoding a [`OreBlockTerm`] encrypts) must exist, fit the chained +/// scheme's 1023 bytes, and not end in NUL. The refusals derivation makes, +/// stated without a key, so a binding can refuse a query at its boundary. +#[cfg_attr(not(feature = "dynamic"), allow(dead_code))] +pub(crate) fn ore_block_text(text: &str) -> Result<(), TermError> { + let folded = vitaminc_ore::text(text)?; + let folded = vitaminc_protected::Controlled::risky_ref(&folded); + if folded.len() > ORE_BLOCK_MAX_BYTES { + return Err(TermError::OreBlockTooLong); + } + if folded.last() == Some(&0) { + return Err(TermError::OreBlockTrailingNul); + } + Ok(()) +} + +/// The longest text or bytes order encoding block ORE takes: ore-rs's +/// chained ceiling (1024 bytes) less the marker byte it puts in front. +const ORE_BLOCK_MAX_BYTES: usize = ore_rs::scheme::chained::DEFAULT_MAX_STR_BYTES - 1; + /// An ORE term of any [`Orderable`] source. Under the local HMAC /// backend, derived during the synchronous build — the returned [`Pending`] /// carries no requests. @@ -1492,6 +1779,30 @@ where } } +/// A block ORE term of any [`Orderable`] source. Under the local HMAC +/// backend, derived during the synchronous build — the returned [`Pending`] +/// carries no requests. +impl<'c, S, K, T> Term> for OreBlockTerm +where + S: Orderable, + T: IntoPrfContext<'c>, +{ + // Derived locally: no data key, no descriptor. + + fn encrypt_from<'a>( + source: S, + cipher: &'a KeysetCipher<'_, K>, + context: NonEmpty, + ) -> Pending<'a, Self, K> + where + Self: 'a, + { + let context = context.into_prf_context().into_owned(); + let term = ore_block(cipher.prf(), source, context).map_err(Error::from); + Pending::ready(cipher, term) + } +} + // ============================================================================= // Term generation on the cipher // ============================================================================= @@ -1659,6 +1970,21 @@ impl KeysetCipher<'_, K> { let term = ope(self.prf(), value, descriptor.into_prf_context()).map(OpeTerm::into_inner); Pending::ready(self, term.map_err(Error::from)) } + + /// Generate a block ORE term: ore-rs's 6-bit block ORE, the scheme + /// EQL v4's `_ord_ore` domains store. Key handling and inputs as for + /// [`ore_term`](Self::ore_term), under a key of its own; see + /// [`OreBlockTerm`] for which scheme a type is encrypted with and what + /// is refused. Byte-identical to the target-directed path's left half + /// for the same descriptor; the right half carries a random nonce. + pub fn ore_block_term<'c, T: Orderable>( + &self, + value: T, + descriptor: NonEmpty>, + ) -> Pending<'_, OreBlockTerm, K> { + let term = ore_block(self.prf(), value, descriptor.into_prf_context()); + Pending::ready(self, term.map_err(Error::from)) + } } #[cfg(test)] diff --git a/packages/stack-encrypt/src/target/index.rs b/packages/stack-encrypt/src/target/index.rs index 08e77c888..5c42fc860 100644 --- a/packages/stack-encrypt/src/target/index.rs +++ b/packages/stack-encrypt/src/target/index.rs @@ -2,7 +2,8 @@ //! engine instead of once per caller. //! //! An *index* is an operation that derives a search term beside a -//! ciphertext: [`Equality`], [`Match`], [`Ore`] or [`Ope`]. Each is a type +//! ciphertext: [`Equality`], [`Match`], [`Ore`], [`Ope`] or [`OreBlock`]. +//! Each is a type //! implementing [`Index`] for exactly the plaintexts its scheme is defined //! over, so an index that does not apply does not compile: `Match` is text //! only, so a match index on an integer is a type error, not a runtime one. @@ -54,11 +55,12 @@ use std::fmt; use std::marker::PhantomData; use super::context::{AeadContext, CallerContext}; -use super::operations::{ciphertext, equality, matching, ope, open, ore}; +use super::operations::{ciphertext, equality, matching, ope, open, ore, ore_block}; use super::source::{ConsumeSource, ShareSource}; use super::{DecryptField, DecryptInto, Decryptable, Decryption, EncryptFrom, Encryption}; use crate::sem::{ - DefaultMatch, EqualityTerm, MatchConfig, MatchOptions, MatchTerms, OpeTerm, OreTerm, + DefaultMatch, EqualityTerm, MatchConfig, MatchOptions, MatchTerms, OpeTerm, OreBlockTerm, + OreTerm, }; use crate::StackCipherText; @@ -69,7 +71,8 @@ use crate::StackCipherText; /// spell an index as an `IndexSpec`, options included. /// /// The [`key`](Self::key) strings are the output keys of the `dynamic` record -/// format (`"eq"`, `"match"`, `"ore"`, `"ope"`), so they are wire format; +/// format (`"eq"`, `"match"`, `"ore"`, `"ope"`, `"ore_block"`), so they are +/// wire format; /// that is why this enum is exhaustive: a new index is something every /// binding has to be taught. With the `dynamic` feature, a plan spells an /// index as its key, or, for a match index with non-default options, as an @@ -81,10 +84,13 @@ pub enum IndexSpec { Equality, /// Full-text match, with the options its terms are generated under. Match(MatchOptions), - /// Order-revealing comparison. + /// Order-revealing comparison, CLLW ORE. Ore, - /// Order-preserving comparison. + /// Order-preserving comparison, CLLW OPE. Ope, + /// Order-revealing comparison, 6-bit block ORE: the term EQL v4's + /// `_ord_ore` domains store as `ob`. + OreBlock, } impl IndexSpec { @@ -101,6 +107,7 @@ impl IndexSpec { IndexSpec::Match(_) => "match", IndexSpec::Ore => "ore", IndexSpec::Ope => "ope", + IndexSpec::OreBlock => "ore_block", } } } @@ -160,6 +167,21 @@ pub struct Ore; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Ope; +/// The block ORE index: ore-rs's 6-bit block ORE ciphertext of the value, +/// under a key derived from the field's context +/// ([`OreBlockTerm`](crate::sem::OreBlockTerm)). Defined for the same `S` +/// as [`Ore`]. +/// +/// [`Ore`] and [`Ope`] are CLLW schemes. This is the other ORE family, the +/// one EQL v4's `_ord_ore` domains store (`ob`), named as EQL names it: an +/// ORE *block* term. Its terms leak less than [`Ore`]'s (a stored term's +/// right half is randomised, and a comparison reveals the first differing +/// six-bit block rather than the first differing bit), cost more (about +/// 300 bytes for an `i64`, 600 for short text), and need a comparator +/// that is not byte order. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct OreBlock; + /// The full-text match index, over text only, with the tokenizer and filter /// parameters its [`MatchConfig`] `O` fixes. /// @@ -265,6 +287,23 @@ where ope() } } +impl Index for OreBlock +where + S: crate::sem::Orderable, +{ + type Term = OreBlockTerm; + fn spec(&self) -> IndexSpec { + IndexSpec::OreBlock + } + fn operation<'s, K: 'static, M: ConsumeSource<'s, S>>( + &self, + ) -> Encryption<'s, S, Self::Term, K, CallerContext, M> + where + S: 's, + { + ore_block() + } +} /// A non-empty set of indexes over `S`: one [`Index`], or a tuple of two to /// four of them. @@ -384,7 +423,7 @@ macro_rules! single_index { } )+}; } -single_index!([] Equality, [O] Match, [] Ore, [] Ope, [] IndexSpec); +single_index!([] Equality, [O] Match, [] Ore, [] Ope, [] OreBlock, [] IndexSpec); /// A tuple of indexes: each index's operation, zipped left to right, and the /// nested pairs `zip` builds flattened back into one tuple of terms. @@ -573,6 +612,13 @@ where type Indexes = Ope; const INDEXES: Ope = Ope; } +impl TermSet for OreBlockTerm +where + S: crate::sem::Orderable, +{ + type Indexes = OreBlock; + const INDEXES: OreBlock = OreBlock; +} /// A tuple of terms names the tuple of their indexes. Each element must be /// one term (its set is one index), as a tuple of indexes holds indexes. diff --git a/packages/stack-encrypt/src/target/mod.rs b/packages/stack-encrypt/src/target/mod.rs index 349cc34ce..897d631f2 100644 --- a/packages/stack-encrypt/src/target/mod.rs +++ b/packages/stack-encrypt/src/target/mod.rs @@ -98,15 +98,15 @@ mod tuples; pub(crate) use self::core::{bind_pending, seal_pending}; pub use context::{AeadContext, CallerContext, DeclaredContext, ExpectedContext, Extends}; pub use index::{ - indexed, At, Encrypted, Equality, Index, IndexSpec, Indexes, Match, Ope, Ore, Select, TermSet, - Whole, + indexed, At, Encrypted, Equality, Index, IndexSpec, Indexes, Match, Ope, Ore, OreBlock, Select, + TermSet, Whole, }; #[cfg(feature = "dynamic")] pub(crate) use operations::chosen; pub(crate) use operations::inspect; pub use operations::{ - ciphertext, equality, matching, ope, open, ore, passthrough, DecryptField, DecryptFrom, - DecryptInto, Decryptable, Decryption, EncryptFrom, EncryptInto, Encryption, + ciphertext, equality, matching, ope, open, ore, ore_block, passthrough, DecryptField, + DecryptFrom, DecryptInto, Decryptable, Decryption, EncryptFrom, EncryptInto, Encryption, }; pub use pending::{CipherScope, Pending, PendingFuture}; pub use request::{Request, Responses}; diff --git a/packages/stack-encrypt/src/target/operations.rs b/packages/stack-encrypt/src/target/operations.rs index cbe026b1e..a8c90c228 100644 --- a/packages/stack-encrypt/src/target/operations.rs +++ b/packages/stack-encrypt/src/target/operations.rs @@ -21,7 +21,7 @@ use std::fmt; /// /// The derive supplies it; a hand-written implementation composes the /// constructors in this module ([`ciphertext`], [`equality`], [`matching`], -/// [`ore`], [`ope`]) and converts their output with [`Encryption::map`] or +/// [`ore`], [`ope`], [`ore_block`]) and converts their output with [`Encryption::map`] or /// [`Encryption::transcode`]. Nothing here receives the plaintext or a /// cipher: the returned [`Encryption`]'s execution is private, so a target /// can choose operations and build its output but cannot replace encryption. @@ -556,6 +556,12 @@ term_operation!( ope, crate::sem::OpeTerm, consume, [], [S: crate::sem::Orderable] ); +term_operation!( + /// The block ORE term of `S` under the context the tree hands it, with + /// the same bounds as [`ore`]. + ore_block, crate::sem::OreBlockTerm, consume, [], + [S: crate::sem::Orderable] +); impl Decryption { /// Reject the opening: execution yields `error` without I/O, any @@ -949,6 +955,21 @@ where vec![super::IndexSpec::Ope] } } +impl EncryptFrom for crate::sem::OreBlockTerm +where + S: crate::sem::Orderable + Clone, +{ + type Context = CallerContext; + fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> + where + S: 's, + { + ore_block() + } + fn indexes() -> Vec { + vec![super::IndexSpec::OreBlock] + } +} // A `Vec` is a row per item, each encrypted independently under the // same context and settled in one batch; an `Option` is one row or diff --git a/packages/stack-encrypt/src/target/transcode.rs b/packages/stack-encrypt/src/target/transcode.rs index 64552fe40..38996a296 100644 --- a/packages/stack-encrypt/src/target/transcode.rs +++ b/packages/stack-encrypt/src/target/transcode.rs @@ -9,7 +9,7 @@ //! What a reader hands over is what the cipher produced, no more: a sealed //! leaf or marker is authenticated ciphertext, but passthrough metadata and //! terms are not, and a visitor must not present them as such. -use crate::sem::{EqualityTerm, MatchConfig, MatchTerms, OpeTerm, OreTerm}; +use crate::sem::{EqualityTerm, MatchConfig, MatchTerms, OpeTerm, OreBlockTerm, OreTerm}; use crate::{CipherText, Error, PassthroughFrame, SealedValue, StackCipherText}; /// An encrypted output that can drive a destination visitor. @@ -120,6 +120,14 @@ pub trait Visitor: Sized { fn ope(self, _: OpeTerm) -> Result { Err(Error::UnsupportedShape) } + /// A block ORE term. One-way, and not authenticated. + /// + /// # Errors + /// + /// Refuses with [`Error::UnsupportedShape`] unless overridden. + fn ore_block(self, _: OreBlockTerm) -> Result { + Err(Error::UnsupportedShape) + } } /// A destination that names its visitor, so an /// [`Encryption`](super::Encryption) can be transcoded into it by type alone. @@ -203,3 +211,8 @@ impl Reader for OpeTerm { visitor.ope(self) } } +impl Reader for OreBlockTerm { + fn read(self, visitor: V) -> Result { + visitor.ore_block(self) + } +} diff --git a/packages/stack-encrypt/tests/ore_block.rs b/packages/stack-encrypt/tests/ore_block.rs new file mode 100644 index 000000000..2168cf10c --- /dev/null +++ b/packages/stack-encrypt/tests/ore_block.rs @@ -0,0 +1,426 @@ +//! Block ORE (`OreBlock`): ore-rs's 6-bit schemes, the terms EQL v4's +//! `_ord_ore` domains store. +//! +//! Every kind the index takes orders as its ADR-0002 order key: integers by +//! value at their own width, floats as PostgreSQL's `float8` (`-0.0` ties +//! `0.0`, every NaN one value after `+Infinity`), text by its fold, bytes +//! as bytes. A term carries a random nonce, so the pins hold the left half +//! only (the header, the permuted symbols and the per-block tags), which is +//! deterministic in the key, the context and the value. +//! +//! Keyed by `FakeDataKeySource`'s deterministic index key, so the pinned +//! bytes are stable without ZeroKMS. + +use std::cmp::Ordering; + +use stack_encrypt::sem::{Orderable, OreBlockTerm, TermBytesError, TermError}; +use stack_encrypt::{nonempty, Error, KeysetCipher, StackCipher}; +use stack_kms::FakeDataKeySource; + +async fn cipher() -> StackCipher { + StackCipher::builder() + .kms(FakeDataKeySource::new()) + .init() + .await + .expect("build cipher") +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +async fn term( + keyset: &KeysetCipher<'_, FakeDataKeySource>, + value: T, +) -> OreBlockTerm { + keyset + .ore_block_term(value, nonempty!("users/n")) + .await + .expect("a block ORE term") +} + +/// The block count of a term, from its header. +fn blocks(term: &[u8]) -> usize { + usize::from(u16::from_be_bytes([term[2], term[3]])) +} + +/// The deterministic left half: the header, then one permuted symbol and +/// one 16-byte tag per block. +fn left(term: &[u8]) -> &[u8] { + let header = if term[1] == 0x02 { 4 } else { 7 }; + &term[..header + 17 * blocks(term)] +} + +/// Every term in `values` (already in plaintext order) compared with every +/// other, both ways round, against the order of its index in the list. +fn assert_sorted(terms: &[OreBlockTerm], what: &str) { + for (i, a) in terms.iter().enumerate() { + for (j, b) in terms.iter().enumerate() { + assert_eq!(a.compare(b), Some(i.cmp(&j)), "{what}: {i} against {j}"); + } + } +} + +macro_rules! orders_integers { + ($keyset:expr, $($ty:ty => $blocks:expr),+ $(,)?) => {$( + let mut values: Vec<$ty> = vec![<$ty>::MIN, <$ty>::MIN + 1, 0 as $ty, 1 as $ty, 63 as $ty, 64 as $ty, <$ty>::MAX - 1, <$ty>::MAX]; + values.sort_unstable(); + values.dedup(); + let mut terms = Vec::new(); + for value in values { + let term = term($keyset, value).await; + assert_eq!(&term.as_bytes()[..2], &[0x02, 0x02], "{} is the 6-bit scheme", stringify!($ty)); + assert_eq!(blocks(term.as_bytes()), $blocks, "{} blocks", stringify!($ty)); + assert_eq!(term.as_bytes().len(), 4 + 25 * $blocks + 16, "{} length", stringify!($ty)); + terms.push(term); + } + assert_sorted(&terms, stringify!($ty)); + )+}; +} + +/// Every integer width from 16 to 64 bits is the 6-bit fixed-width scheme, +/// one block per six bits of the width, and orders by value, negatives +/// first. (An 8-bit integer has no order term.) +#[tokio::test] +async fn integers_order_by_value_at_their_width() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + orders_integers!(&keyset, + i16 => 3, u16 => 3, i32 => 6, u32 => 6, i64 => 11, u64 => 11, + ); + let negatives = [-1_000_000i32, -65, -64, -63, -1, 0, 1]; + let mut terms = Vec::new(); + for value in negatives { + terms.push(term(&keyset, value).await); + } + assert_sorted(&terms, "negative i32"); +} + +/// The 128-bit integers are wider than the 6-bit scheme's ten bytes, so +/// they go through the chained scheme, unpadded: 22 blocks each. +#[tokio::test] +async fn wide_integers_are_the_chained_scheme_unpadded() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let mut signed = Vec::new(); + for value in [i128::MIN, -1, 0, 1, i128::MAX] { + let term = term(&keyset, value).await; + assert_eq!(&term.as_bytes()[..2], &[0x02, 0x03]); + assert_eq!(&term.as_bytes()[4..7], &[0, 0, 0], "no padding"); + assert_eq!(blocks(term.as_bytes()), 22); + signed.push(term); + } + assert_sorted(&signed, "i128"); + let mut unsigned = Vec::new(); + for value in [0u128, 1, u128::MAX] { + unsigned.push(term(&keyset, value).await); + } + assert_sorted(&unsigned, "u128"); +} + +/// Floats order as PostgreSQL's `float8`: `-0.0` ties `0.0`, every NaN is +/// one value after `+Infinity`, and an `f32` is the `f64` it widens to. +#[tokio::test] +async fn floats_follow_the_semantic_baseline() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let ordered = [ + f64::NEG_INFINITY, + f64::MIN, + -1.5, + -f64::MIN_POSITIVE, + 0.0, + f64::MIN_POSITIVE, + 1.5, + f64::MAX, + f64::INFINITY, + f64::NAN, + ]; + let mut terms = Vec::new(); + for value in ordered { + let term = term(&keyset, value).await; + assert_eq!(blocks(term.as_bytes()), 11, "an f64 is eight bytes"); + terms.push(term); + } + assert_sorted(&terms, "f64"); + assert_eq!( + term(&keyset, -0.0f64) + .await + .compare(&term(&keyset, 0.0f64).await), + Some(Ordering::Equal) + ); + let negative_nan = f64::from_bits(f64::NAN.to_bits() | (1 << 63)); + assert_eq!( + term(&keyset, negative_nan) + .await + .compare(&term(&keyset, f64::NAN).await), + Some(Ordering::Equal) + ); + for value in [-1.5f32, 0.1, f32::MAX, f32::NAN] { + let narrow = term(&keyset, value).await; + let wide = term(&keyset, f64::from(value)).await; + assert_eq!(narrow.as_bytes()[..2], wide.as_bytes()[..2]); + assert_eq!(left(narrow.as_bytes()), left(wide.as_bytes()), "{value:?}"); + } +} + +/// Text is the chained scheme, padded to a power of two of at least 16 +/// bytes (the marker byte counts): a term reveals the bucket, not the +/// length. Shared prefixes, different buckets and the empty string order +/// as their folds do, shorter first. +#[tokio::test] +async fn text_orders_by_its_fold_shortest_first() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let fifteen = "a".repeat(15); + let sixteen = "a".repeat(16); + let thirty_one = "a".repeat(31); + let thirty_two = "a".repeat(32); + let ordered: Vec = vec![ + String::new(), + "a".into(), + "aa".into(), + fifteen.clone(), + sixteen.clone(), + thirty_one.clone(), + thirty_two.clone(), + "ab".into(), + "abc".into(), + "abd".into(), + "alice".into(), + "b".into(), + "Bob".into(), + "zoe".into(), + ]; + let mut terms = Vec::new(); + for value in &ordered { + let term = term(&keyset, value.clone()).await; + assert_eq!(&term.as_bytes()[..2], &[0x02, 0x03]); + assert_eq!(&term.as_bytes()[4..7], &[1, 0, 16], "powers of two from 16"); + terms.push(term); + } + assert_sorted(&terms, "text"); + + // The bucket is the length of the marker and the fold, rounded up. + for (text, count) in [ + ("", 22), + (fifteen.as_str(), 22), + (sixteen.as_str(), 43), + (thirty_one.as_str(), 43), + (thirty_two.as_str(), 86), + ] { + let term = term(&keyset, text.to_string()).await; + assert_eq!(blocks(term.as_bytes()), count, "{} bytes", text.len()); + assert_eq!(term.as_bytes().len(), 7 + 25 * count + 16); + } + + // Case and accents fold: these tie. + let cafe = term(&keyset, "cafe".to_string()).await; + for spelling in ["Café", "CAFE", "cafe\u{301}"] { + assert_eq!( + term(&keyset, spelling.to_string()).await.compare(&cafe), + Some(Ordering::Equal), + "{spelling:?}" + ); + } +} + +/// Bytes are the chained scheme unpadded, ordered as bytes: a trailing zero +/// byte is kept, and the empty value still makes a keyed block. +#[tokio::test] +async fn bytes_order_as_bytes() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let ordered: Vec> = vec![vec![], vec![0], vec![0, 0], vec![1], vec![1, 0], vec![255]]; + let mut terms = Vec::new(); + for value in ordered { + let term = term(&keyset, value).await; + assert_eq!(&term.as_bytes()[4..7], &[0, 0, 0], "bytes are not padded"); + assert!( + blocks(term.as_bytes()) >= 2, + "the marker byte is two blocks" + ); + terms.push(term); + } + assert_sorted(&terms, "bytes"); +} + +/// One value encrypts to different bytes each time (the right half's +/// nonce), and those terms compare equal; the left half is deterministic. +#[tokio::test] +async fn one_value_is_new_bytes_that_compare_equal() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + for (a, b) in [ + ( + term(&keyset, 42i64).await.into_bytes(), + term(&keyset, 42i64).await.into_bytes(), + ), + ( + term(&keyset, "alice".to_string()).await.into_bytes(), + term(&keyset, "alice".to_string()).await.into_bytes(), + ), + ] { + assert_ne!(a, b, "a fresh nonce each time"); + assert_eq!(left(&a), left(&b)); + let (a, b) = ( + OreBlockTerm::::from_bytes(&a).unwrap(), + OreBlockTerm::::from_bytes(&b).unwrap(), + ); + assert_eq!(a.compare(&b), Some(Ordering::Equal)); + assert!(a == b, "== is the scheme's equality"); + } +} + +/// Every view of a term's bytes is the one serialised ciphertext, and its +/// `Debug` names the length only. +#[tokio::test] +async fn every_view_of_a_term_is_its_ciphertext() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let below = term(&keyset, 41i64).await; + let term = term(&keyset, 42i64).await; + // `PartialOrd` is the scheme's comparator. + assert!(below < term); + assert!(term >= below); + assert_eq!(below.partial_cmp(&below.clone()), Some(Ordering::Equal)); + assert_eq!(term.partial_cmp(&below), Some(Ordering::Greater)); + let bytes = term.as_bytes().to_vec(); + assert_eq!(bytes[..2], [0x02, 0x02], "the fixed-width scheme's header"); + assert_eq!(term.to_bytes(), bytes); + assert_eq!(AsRef::<[u8]>::as_ref(&term), bytes.as_slice()); + assert_eq!( + format!("{term:?}"), + format!("OreBlockTerm {{ len: {} }}", bytes.len()) + ); + assert_eq!(term.into_bytes(), bytes); +} + +/// The deterministic left halves, pinned: a change to the key derivation, +/// the salt, the scheme selection or ore-rs's wire format changes every +/// stored term and must fail here. +#[tokio::test] +async fn left_halves_are_pinned() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let i32_term = keyset + .ore_block_term(42i32, nonempty!("users/age")) + .await + .unwrap(); + let f64_term = keyset + .ore_block_term(1.5f64, nonempty!("users/score")) + .await + .unwrap(); + let text_term = keyset + .ore_block_term("Ab".to_string(), nonempty!("users/surname")) + .await + .unwrap(); + assert_eq!( + [ + hex(left(i32_term.as_bytes())), + hex(left(f64_term.as_bytes())), + hex(left(text_term.as_bytes())) + ], + [PINNED_I32, PINNED_F64, PINNED_TEXT] + ); +} + +const PINNED_I32: &str = "020200060c0a0a1f37375535b8d860bd115a46292e8aac8ef6571e4ab59087b3b4c68d4621980b5633ca7f5a6d0da5d7d9d683003c2bfcc4630697cbce320a75f61b0875bc8d63c530d74c515d0ee6fc1b5303f5ff16e767ad905a95a1e5fbfebfb047d2f959b91b9b3d"; +const PINNED_F64: &str = "0202000b1501163b2a0a33341205014148b5ed0af663b698a1e4727de4a2f9749c116d8be814ed51540319016cc48b5c9ca000d776bf2aa438a615f84fe4c8856567731aebe1d043dcee1cf0a62e1995e3247033d370b0c7b24e6c043d33ecff732552dacfd13cad5b7b11f5c90f6480e05c5d75b04d605a7522c5102491999f35690dcd5ae228b08e812766bf35f8622307ffce17091ad737af98f1c82a9caa2fa556156c52ee7caee7b97d2b6a8f6e0e1ac247e006dae5d61fee390b39e2"; +const PINNED_TEXT: &str = "0203001601001039201c002d22313e3625260c211a36351e3507391835d9cec533ab14188edbe7bb248118b82ff3a1635b1469dd20ab6021b66d5d957893d251ad85f93322d5d445a149f4cea13a7be3692c56241a6cf438a1b6f3353bac005c4eabbeb193ee467a15d7b5cf6d171d8347f4fe2a9d87694bcc86db61747b9e4a1f76c078331be8e57291969b4c6a9e38d6efa1500d921c712888fa953fc94099f0cc0a5852c1decc45563122685a3c07184f4f49ed2b98abb2682e22fcb42140d0a0ff51cca79c678a0b0077c7de2f6690b6e63957e433a96e4f42b2f6e8eb15180349c8d3a4aa9e3663e11e90267b22428407a5f051095113a0ac0bbcbf99e0c4fc41fe2f1c233cf27af369a5f956f29813a6a153f78ab56af7e60f06f3a8fc7a637e6749d44218c0a58d5541944c85a970aff3ca00dbe3bc6be3fdd1944ccab358071e8baac6aeec381c2a2fdac3ba2d69b3c09f7694abf699f6a652da1a2510df98ac20716dd7c25f834d412c0617d195922952942187e399bb6b6d"; + +/// The block ORE key is the index's own: its terms share no left half with +/// another context's, and none with CLLW ORE's key (a CLLW term is a +/// different shape altogether, so this checks the contexts). +#[tokio::test] +async fn terms_are_keyed_by_their_context() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let users = keyset + .ore_block_term(7i32, nonempty!("users/age")) + .await + .unwrap(); + let orders = keyset + .ore_block_term(7i32, nonempty!("orders/age")) + .await + .unwrap(); + assert_ne!(left(users.as_bytes()), left(orders.as_bytes())); +} + +/// What block ORE refuses, each without naming the value: text whose fold +/// is longer than 1023 bytes, text that ends in NUL, text the fold cannot +/// take, and a boolean. +#[tokio::test] +async fn refusals_name_no_part_of_the_value() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let refused = |error: Error| match error { + Error::Term(error) => error, + other => panic!("not a term refusal: {other:?}"), + }; + let long = "secret".repeat(200); + assert!(keyset + .ore_block_term("a".repeat(1023), nonempty!("users/n")) + .await + .is_ok()); + let too_long = refused( + keyset + .ore_block_term(long.clone(), nonempty!("users/n")) + .await + .unwrap_err(), + ); + assert!( + matches!(too_long, TermError::OreBlockTooLong), + "{too_long:?}" + ); + assert!(!too_long.to_string().contains("secret")); + let nul = refused( + keyset + .ore_block_term("secret\0".to_string(), nonempty!("users/n")) + .await + .unwrap_err(), + ); + assert!(matches!(nul, TermError::OreBlockTrailingNul), "{nul:?}"); + assert!(!nul.to_string().contains("secret")); + let unassigned = refused( + keyset + .ore_block_term("a\u{378}b".to_string(), nonempty!("users/n")) + .await + .unwrap_err(), + ); + assert!( + matches!(unassigned, TermError::Normalize(_)), + "{unassigned:?}" + ); + // A boolean and an 8-bit integer have no block ORE term, as they have + // no ORE or OPE term: `ore_block_term(true, ..)` and + // `ore_block_term(7u8, ..)` do not compile (`sem::Orderable`). +} + +/// Decoding is structural: a ciphertext of either scheme decodes, and +/// anything else, including a truncated one, is refused with its length. +#[tokio::test] +async fn decoding_checks_the_shape() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let bit6 = term(&keyset, 42i64).await.into_bytes(); + let chained = term(&keyset, "alice".to_string()).await.into_bytes(); + assert!(OreBlockTerm::::from_bytes(&bit6).is_ok()); + assert!(OreBlockTerm::::from_bytes(&chained).is_ok()); + for bad in [ + &bit6[..bit6.len() - 1], + &chained[..10], + &[][..], + &[0x02, 0x04, 0, 1][..], + ] { + assert_eq!( + OreBlockTerm::::from_bytes(bad).unwrap_err(), + TermBytesError::MalformedOreBlockCiphertext(bad.len()) + ); + } + // A 6-bit and a chained term have no order between them. + let a = OreBlockTerm::::from_bytes(&bit6).unwrap(); + let b = OreBlockTerm::::from_bytes(&chained).unwrap(); + assert_eq!(a.compare(&b), None); + assert!(a != b); +} diff --git a/packages/stack-encrypt/tests/plan_grammar.rs b/packages/stack-encrypt/tests/plan_grammar.rs index cd5ceca46..1fc6067f7 100644 --- a/packages/stack-encrypt/tests/plan_grammar.rs +++ b/packages/stack-encrypt/tests/plan_grammar.rs @@ -15,7 +15,7 @@ use stack_encrypt::kms::FakeDataKeySource; use stack_encrypt::op; use stack_encrypt::plan::{pick, Field, FieldKind, FieldValues, Fields, Opens, PlanError, Runs}; use stack_encrypt::sem::{ - EqualityTerm, MatchConfig, MatchOptions, MatchTerms, OpeTerm, OreTerm, Tokenizer, + EqualityTerm, MatchConfig, MatchOptions, MatchTerms, OpeTerm, OreBlockTerm, OreTerm, Tokenizer, }; use stack_encrypt::target::{ CallerContext, DeclaredContext, DecryptField, DecryptInto, Decryptable, Decryption, @@ -1727,6 +1727,10 @@ fn every_target_declares_the_indexes_its_terms_answer() { as EncryptFrom>::indexes(), [IndexSpec::Ope] ); + assert_eq!( + as EncryptFrom>::indexes(), + [IndexSpec::OreBlock] + ); assert!(>::indexes().is_empty()); assert_eq!( , EqualityTerm)> as EncryptFrom>::indexes(),