From 760e5bdcc62f2fe1e4ab0336d30277a0d5a6bbb6 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 13:12:56 +1100 Subject: [PATCH 1/7] test(eql): hold the planner to the json_tree indexes on every supported PostgreSQL, and document what the tree reveals Two of the three items that gate a release of EQL v4's json_tree. The planner. `eql_v4.json_tree_query` is plpgsql with `COST 1000` so that a bitmap scan over the GIN index beats a sequential scan even when the documents are TOASTed and the heap is a few pages. The benches showed that on one PostgreSQL version and nothing held it: the oracle test checks the plan with sequential scans turned off, which is not what a customer's database runs. The new test seals a flat table (1,000 four-node documents, every row inline) and a nested one (300 documents of about 220 nodes, every row TOASTed), pins both regimes by `pg_column_size`, runs `ANALYZE`, and with every setting at its default asserts a bitmap scan over the GIN index for containment and a scan of the functional btree for a range at an ordered path. A control prices the function at 1 for one transaction and shows the sequential scan the cost guards against, so the assertion is known to be able to fail. CI now runs the Postgres half of the encryption suite in its own job, a matrix over the supported PostgreSQL versions (17 on a pull request, 14 to 17 on a push, like the SQLx jobs), where it was one step of the single-version `rust-crates` job. Passed locally on 14, 15, 16 and 17. The page. `docs/reference/json-tree-leakage.md` is the one page a customer reads before putting a column on the type: what a stored row shows at rest, field by field; what each query type hands the database, tokens above all; what is authenticated; the side-by-side with v3; and the two options that change the picture. Refs #1324 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../eql-json-tree-planner-and-leakage.md | 5 + .github/workflows/test-eql.yml | 90 +++++-- ...26-10-17-json-index-stack-encrypt-spike.md | 7 + packages/eql/docs/README.md | 1 + .../eql/docs/reference/json-tree-leakage.md | 60 +++++ .../eql/tests/encryption/tests/json_tree.rs | 235 ++++++++++++++++++ 6 files changed, 374 insertions(+), 24 deletions(-) create mode 100644 .changeset/eql-json-tree-planner-and-leakage.md create mode 100644 packages/eql/docs/reference/json-tree-leakage.md diff --git a/.changeset/eql-json-tree-planner-and-leakage.md b/.changeset/eql-json-tree-planner-and-leakage.md new file mode 100644 index 000000000..b08b2b923 --- /dev/null +++ b/.changeset/eql-json-tree-planner-and-leakage.md @@ -0,0 +1,5 @@ +--- +"@cipherstash/eql": patch +--- + +EQL v4 `json_tree` (unreleased): the reference docs gain "What an encrypted JSON tree reveals" (`docs/reference/json-tree-leakage.md`), one page on what a stored row shows the database at rest, what each query type hands it, what is authenticated, and the side-by-side with v3 `ste_vec`. The encryption test suite now holds the planner to the GIN and btree indexes with every setting at its default, on inline and TOASTed documents, and CI runs that suite once per supported PostgreSQL version. diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index e0b1f16be..35034b7b6 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -452,6 +452,68 @@ jobs: # docs:validate + Clean-DB v3 install smoke. Both are version-relevant, so # they follow the event's PG set (PG17 on PR; 14-17 in the queue). Moved out # of the old per-version test job so they run ONCE per version, not per shard. + # The Rust encryption suite against a live database, once per supported + # PostgreSQL version: the JSON tree family's oracle (containment over the + # encrypted column against native `jsonb @>` over the plaintexts), its + # walks and orderings, and the plan-shape test that holds the planner to + # the GIN and btree indexes with every setting at its default. Plan choice + # is a per-version property of the planner, which is why this is a matrix + # and not a step of `rust-crates`. PR and manual dispatch run PG 17 alone; + # a push runs the full set, like the SQLx jobs (see `setup`). + rust-encryption-postgres: + name: "Rust encryption with PostgreSQL ${{ matrix.postgres-version }}" + needs: [changes, setup] + # Exclusion, not an allowlist — see the note on `build-archive`. + if: >- + github.event_name != 'pull_request' + || needs.changes.outputs.relevant == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + strategy: + fail-fast: false + matrix: + postgres-version: ${{ fromJSON(needs.setup.outputs.pg-versions) }} + services: + encryption-postgres: + image: postgres:${{ matrix.postgres-version }} + env: + POSTGRES_PASSWORD: postgres + # Container port only: Docker picks the host port, and the test step + # reads it from `job.services`, so the number is written once. + ports: + - 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + cache: true + working_directory: packages/eql + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2204 + # Restores what `rust-crates` saves (the stack-encrypt feature tree) + # and never saves itself: a matrix of fast jobs racing one key would + # overwrite it with whichever finished first. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: packages/eql + shared-key: rust-crates + save-if: false + # The task takes the connection from EQL_TEST_DATABASE_URL alone, and + # the host port in it comes from the service, so the two cannot + # disagree. + - name: Test Rust encryption with PostgreSQL ${{ matrix.postgres-version }} + env: + EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.encryption-postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres" + run: mise run test:encryption:postgres + validate: name: "Validate (Postgres ${{ matrix.postgres-version }})" needs: [changes, setup] @@ -548,20 +610,6 @@ jobs: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' runs-on: blacksmith-16vcpu-ubuntu-2204 - services: - encryption-postgres: - image: postgres:17 - env: - POSTGRES_PASSWORD: postgres - # Container port only: Docker picks the host port, and the test step - # reads it from `job.services`, so the number is written once. - ports: - - 5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -603,20 +651,14 @@ jobs: - name: Check the timing harness's Cargo.lock run: cargo metadata --locked --format-version 1 --manifest-path tests/timing/ore_block_v2/ctgen/Cargo.toml > /dev/null - # The three encryption checks are mise tasks rather than raw cargo here, - # so `eql-suite-ci.test.mjs` counts them among the cargo tasks a root + # The encryption checks are mise tasks rather than raw cargo here, so + # `eql-suite-ci.test.mjs` counts them among the cargo tasks a root # workflow must reach, and `eql-workflow-filters.test.mjs` derives the - # paths they read. The database task takes the connection from - # EQL_TEST_DATABASE_URL alone, and the host port in it comes from the - # service, so the two cannot disagree. + # paths they read. The database half runs in `rust-encryption-postgres` + # below, once per supported PostgreSQL version. - name: Test Rust text equality run: mise run test:encryption - - name: Test Rust text equality with PostgreSQL - env: - EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.encryption-postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres" - run: mise run test:encryption:postgres - - name: Compile Rust text equality for WASI without HTTP run: mise run check:encryption:wasi diff --git a/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md b/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md index 38064caee..67c154545 100644 --- a/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md +++ b/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md @@ -477,6 +477,13 @@ The rows were taken from the Phase 3c tables (one ordered path a corpus), re-enc The nested document is now about 14 times its plaintext on disk (18 at Phase 3c, 40 at Phase 3). The byte saving exceeds the character saving: pglz compressed hex better than it compresses base64 and base85, but not by enough to matter. The benchmark corpora are not re-ingested and no query is re-timed in this note; the walk, containment and ordering paths are covered by the oracle suite, which passed against the bundle built from the tree. Of the two levers left from the Phase 3 list, one is closed and one is open. **The derived nonce is rejected (2026-10-23).** It would save the twelve random bytes in every `c`, about 7% of a nested document, but it needs vitaminc to expose a caller-chosen nonce (a nonce strategy that reads the leaf's AAD, and a nonce-less leaf wire form), and a caller-chosen nonce is exactly what vitaminc's AEAD layer refuses to offer: its misuse resistance rests on never letting a caller repeat a (key, nonce) pair, and the safety argument here ("every seal mints a fresh data key") is an invariant of stack-encrypt that vitaminc could not check. Seven percent does not buy weakening that promise; the random nonce stays. Unpadded base64 (two characters a tag, about 3%) remains open and untaken. +## Implementation notes (2026-10-23, Phase 4: the planner held to its indexes, and the leakage page) + +Two of the three items that gate a release are closed here; the third, the final re-bench, follows them. + +- **Planner fragility (gate 2).** `eql_v4.json_tree_query` is plpgsql with `COST 1000` so that a bitmap scan over the GIN index beats a sequential scan even when the documents are TOASTed and the heap looks like a few pages. That was observed in the benches, at 10,000 and 100,000 rows, on one PostgreSQL version, and nothing held it. `postgres_the_planner_picks_the_indexes_without_help` in the encryption suite now does: on a flat table (1,000 four-node documents, every row inline) and a nested table (300 documents of about 220 nodes, every row TOASTed, both regimes pinned by `pg_column_size`), with every planner setting at its default and after `ANALYZE`, containment plans as a bitmap scan over the GIN index and a range at an ordered path as a scan of the functional btree on `ord_term(doc -> selector)`. A control sets the function's cost to 1 for one transaction and shows the planner taking the sequential scan on the nested table, so the assertion is known to be able to fail. The suite runs in `test-eql.yml` in a new `rust-encryption-postgres` job, a matrix over the supported PostgreSQL versions (17 on a pull request, 14 to 17 on a push, like the SQLx jobs); it was a step of the single-version `rust-crates` job. Measured locally on 14, 15, 16 and 17 before the commit. +- **Leakage (gate 3).** `packages/eql/docs/reference/json-tree-leakage.md` is the one page: what a stored row shows at rest, field by field; what each query type hands the database; what is authenticated; the side-by-side with v3 from section 6 above, brought up to date (no `g`, position-keyed `p`, ordering opt-in); and the two options that change the picture. It says what a token is and that it is column-wide and does not expire, which is the sentence a customer most needs before sending one. + ## Decisions to take before Phase 1 > **2026-10-22:** these are the open questions as they stood before Phase 1; the implementation notes above record how each was taken. Since then, decision 4's group tags exist only as the nesting of the stored projection `q`, decision 9 was settled by storing the projection (Phase 3), and decision 10's ordinal is not stored (Phase 3b). diff --git a/packages/eql/docs/README.md b/packages/eql/docs/README.md index 8b7a6c1d9..b5b9d7e84 100644 --- a/packages/eql/docs/README.md +++ b/packages/eql/docs/README.md @@ -14,6 +14,7 @@ This directory contains the documentation for the Encrypt Query Language (EQL). - [Writing fast queries against EQL columns](reference/query-performance.md) - Performance overview (points to Database Indexes) - [Adding a Scalar Encrypted-Domain Type](reference/adding-a-scalar-encrypted-domain-type.md) - How the `eql_v3.` domain families are generated - [EQL with JSON and JSONB](reference/json-support.md) +- [What an encrypted JSON tree reveals](reference/json-tree-leakage.md) - What `public.eql_v4_json_tree` (EQL v4) shows the database at rest and at query time, and what it does not - [EQL payload / wire format](../crates/eql-bindings/README.md) - Canonical wire types for the encrypted payload (envelope `v`/`i`/`c` and the `hm`/`op`/`ob`/`bf` index terms) - [Client-side index configuration](https://cipherstash.com/docs/stack/cipherstash/encryption/schema) - Configuring searchable encryption in CipherStash Stack / CipherStash Proxy diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md new file mode 100644 index 000000000..9c87b8f15 --- /dev/null +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -0,0 +1,60 @@ +# What an encrypted JSON tree reveals + +`public.eql_v4_json_tree` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page reveals one. What the database can see is a bounded amount of **structure**, and it is deliberate: each item below is what one query capability costs. Read this before you put a column on it, and decide the two options at the end with it in mind. + +## On this page + +- [What a stored row shows](#what-a-stored-row-shows) +- [What a query shows](#what-a-query-shows) +- [What is authenticated](#what-is-authenticated) +- [Compared with EQL v3 (`public.eql_v3_json_search`)](#compared-with-eql-v3-publiceql_v3_json_search) +- [The two options that change the picture](#the-two-options-that-change-the-picture) + +## What a stored row shows + +Someone who holds the column and no key, such as a database administrator, a backup, or an attacker with a copy of the table, learns the following from each row. "Across rows" means a comparison between rows of the same column; nothing links a column to another column or another keyset. + +| field | what it reveals | +|---|---| +| `s`, the path selector on every node | **Which paths the row has.** A selector is a keyed hash of the path, the same in every row of the column, so two rows with a selector in common hold the same path. Key names are inside the hash and not readable. Array elements share their array's selector, so `$.tags[0]` and `$.tags[1]` look alike. | +| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value. Not the value itself, and not whether two different paths hold the same value. **Which nodes sit inside the same array element**, through the nesting. That grouping is what makes containment exact for arrays of objects, and it is the one thing this format reveals that v3 did not. | +| the number of entries, and each ciphertext's length | **The node count**, and for each scalar the length of its value plus its key name, to within the AEAD's fixed overhead. A row's total size is the usual size signal any encrypted column gives. | +| `p`, the parent tag below the chain's heads | Nothing on its own. It is salted by the document's `iv`, so the same position in two rows is unrelated bytes. It shows **how many nodes are direct members of the root** (they carry no `p`); not which node is any other node's parent, not an array's length, and not whether two rows have the same shape. | +| `op`, the ordering term, only on leaves at the paths the index orders | **The order of that leaf's value among every row that holds one.** The term is order-preserving encryption under one key per column, so leaves at two ordered paths compare with each other too. An order-preserving term is not a random value: values that sort near each other get terms that share leading bytes. By default no path is ordered; see the options below. | + +Not revealed at rest: any value, any key name, which node is whose parent, how long an array is, and whether two rows have the same shape. + +## What a query shows + +A query hands the database a term derived from the plaintext it asks about, and the database's answer is visible to it. Each query type reveals the following, in addition to the rows it returns. + +- **Containment (`@>`, `<@`).** The needle's own projection: the selectors of its paths, the value selectors of its leaves, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. +- **Path extraction (`->`, `->>`, `json_tree_path_query`).** The selector of the path asked. The entry returned is a ciphertext. +- **Ordering at a path (`<`, `<=`, `>`, `>=`, `ORDER BY ord_term(…)`).** The bound's own ordering term, so roughly where the bound sits in the order of the stored values at that path. +- **A subtree read (`json_tree_subtree`, `json_tree_subtree_at`, `json_tree_children`, `json_tree_child_count`).** A **token** for the node, and one gate (a selector or an array position) per step below it. A token is a capability over the structure of that subtree in *every row of the column*: whoever holds it can count the nodes under that path in any row, see how they nest and in what order, and derive the tokens of the nodes beneath. It still opens no value. Tokens are column-wide and do not expire, so a token sent once is useful to the database for as long as the column lives. Send a token to the database in a query and nowhere else. With the sealed root on (the default) the broadest token a query can send covers one top-level field's subtree; no query can walk a whole column. + +## What is authenticated + +Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under this data key, at this path (`s`) and at this position in the document (`j`); a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the ordering terms, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). The consequence to design around: tampering with those fields changes **which rows a query returns or where a walk stops**, never what a value decrypts to. A query's result set is not integrity-protected; a decrypted value is. + +## Compared with EQL v3 (`public.eql_v3_json_search`) + +| observer sees | v3 (`ste_vec`) | v4 (`json_tree`) | +|---|---|---| +| which paths a row has | yes | yes | +| which (path, value) pairs a row has | yes | yes | +| node count | yes, two entries per node | yes, one entry per node | +| size of every subtree | yes: each path entry's ciphertext is the subtree under it | no: only each scalar's length | +| parent and child relation | inferable from subtree sizes and co-occurrence across rows | hidden at rest; linkable only with a token the query supplies | +| whether two rows have the same shape | yes, selectors are column-deterministic | no, until a token is supplied | +| which leaves sit in the same array element | no | yes, through the projection's nesting: the one new disclosure, required for exact containment, bounded to array elements | +| equal values in sibling elements | ciphertexts equal | ciphertexts differ | +| ordering terms | on scalar leaves, with no per-path choice | only at the paths the index orders; none by default | +| a subtree read | reveals the selector asked | reveals the token asked, and with it the subtree's shape in every row for the life of the column | + +## The two options that change the picture + +- **`sealed_root`** (default on). The root entry's ciphertext is the whole document, so a full read is one decryption and the broadest token a query can send is one top-level field's. The cost is that the root entry's length reveals the document's size, and the row is larger. +- **`ordered`** (default none). Each path listed carries an ordering term and takes on the `op` row above; a path not listed cannot be range-queried or sorted, and a query that tries is refused by the client before any key loads. Order only the paths your queries sort or range over. + +To report a concern with any of this, see `SECURITY.md` at the repository root. diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index b73168172..05b2b5db2 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -792,3 +792,238 @@ fn postgres_extracts_walks_and_orders_by_path() { } } } + +/// A document of the flat shape: four nodes, under PostgreSQL's TOAST +/// threshold, so the row sits inline in the heap. +fn flat_document(i: i64) -> Value { + json!({"number": i, "hello": format!("h{}", i % 17), "flag": i % 2 == 0}) +} + +/// A document of the nested shape: about 220 nodes, well over the TOAST +/// threshold, so the heap holds a pointer and the document sits in the +/// TOAST relation. +fn nested_document(i: i64) -> Value { + let orders: Vec = (0..12) + .map(|k| { + let items: Vec = (0..5) + .map(|m| json!({"sku": format!("s{}", (i + k + m) % 23), "qty": m + 1, "gift": m == 0})) + .collect(); + json!({"id": i * 100 + k, "total": (i + k) % 997, "items": items}) + }) + .collect(); + json!({"number": i, "hello": format!("h{}", i % 17), "orders": orders}) +} + +/// Needs an EQL v4 installation. The planner chooses the indexes on its +/// own, with every setting at its default, on both shapes a stored +/// document takes: inline (a small document, under the TOAST threshold) +/// and TOASTed (a large one, where the heap holds a pointer and a +/// sequential scan looks cheap by page count). `json_tree_query` is +/// plpgsql with `COST 1000` for exactly that reason: the planner cannot +/// see that evaluating it detoasts the row, so its cost has to say so. +/// The oracle test above shows the GIN index with sequential scans turned +/// off; this one shows it with them on, which is what a customer's +/// database runs. The ordering path is held the same way: a functional +/// btree on `ord_term(doc -> selector)` serves a range at a path. +/// +/// The last block is a control: with the function's cost set to the +/// minimum for one transaction, the planner prefers the sequential scan +/// on the TOASTed table, which is what the assertion above guards +/// against. Read it as "the assertion can fail", not as a target. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_planner_picks_the_indexes_without_help() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let plan = plan(); + let mut db = connect(); + + let version: i32 = db + .query_one("SHOW server_version_num", &[]) + .unwrap() + .get::<_, String>(0) + .parse() + .unwrap(); + + // The operands: a containment needle per shape, and one range bound at + // `$.number`, whose selector the functional index is built on. + let bound = wait( + &runtime, + cipher + .query(op::AtPath(op::Gte), &("$.number", 1)) + .on(&plan), + ) + .unwrap(); + let number = bound.selector().to_string(); + let leaf = |value: i64| { + let query = wait( + &runtime, + cipher + .query(op::AtPath(op::Gte), &("$.number", value)) + .on(&plan), + ) + .unwrap(); + serde_json::to_string(&JsonTreeLeafQuery::try_from(&query).unwrap()).unwrap() + }; + let needle = |value: Value| { + let query = wait(&runtime, cipher.query(op::Contains, &value).on(&plan)).unwrap(); + serde_json::to_string(&JsonTreeQuery::try_from(&query).unwrap()).unwrap() + }; + + let explain = + |db: &mut postgres::Client, sql: &str, params: &[&(dyn postgres::types::ToSql + Sync)]| { + db.query(&format!("EXPLAIN {sql}"), params) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get::<_, String>(0)) + .collect::>() + .join("\n") + }; + + struct Shape { + name: &'static str, + rows: i64, + document: fn(i64) -> Value, + needle: Value, + inline: bool, + } + let shapes = [ + Shape { + name: "flat", + rows: 1000, + document: flat_document, + needle: json!({"hello": "h3"}), + inline: true, + }, + Shape { + name: "nested", + rows: 300, + document: nested_document, + needle: json!({"orders": [{"items": [{"sku": "s7", "gift": true}]}]}), + inline: false, + }, + ]; + + for shape in &shapes { + let table = format!("json_tree_plan_{}", shape.name); + db.batch_execute(&format!( + "CREATE TEMP TABLE {table} (id int PRIMARY KEY, doc public.eql_v4_json_tree NOT NULL); + CREATE INDEX {table}_gin ON {table} USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops); + CREATE INDEX {table}_ord ON {table} (eql_v4.ord_term(doc -> '{number}'::text));" + )) + .unwrap(); + for i in 0..shape.rows { + let stored = wait(&runtime, seal(&cipher, &(shape.document)(i))); + let stored = serde_json::to_string(&stored).unwrap(); + db.execute( + &format!( + "INSERT INTO {table} VALUES ($1, $2::text::jsonb::public.eql_v4_json_tree)" + ), + &[&(i as i32), &stored], + ) + .unwrap(); + } + db.batch_execute(&format!("ANALYZE {table}")).unwrap(); + + // The two storage regimes the shapes stand for, pinned, so the test + // keeps meaning when a document's size changes. + let (column, toasted): (i32, i64) = { + let row = db + .query_one( + &format!( + "SELECT avg(pg_column_size(doc))::int, + (SELECT count(*) FROM {table} WHERE pg_column_size(doc) > 2000) + FROM {table}" + ), + &[], + ) + .unwrap(); + (row.get(0), row.get(1)) + }; + if shape.inline { + assert_eq!( + toasted, 0, + "{}: every row inline, {column} bytes on average", + shape.name + ); + } else { + assert_eq!( + toasted, shape.rows, + "{}: every row TOASTed, {column} bytes on average", + shape.name + ); + } + + // Containment: a bitmap scan over the GIN index, on its own. + let operand = needle(shape.needle.clone()); + let text = explain( + &mut db, + &format!("SELECT id FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree"), + &[&operand], + ); + assert!( + text.contains(&format!("Bitmap Index Scan on {table}_gin")), + "{} on PostgreSQL {version}: containment should use the GIN index without help:\n{text}", + shape.name + ); + let matched: i64 = db + .query_one( + &format!("SELECT count(*) FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree"), + &[&operand], + ) + .unwrap() + .get(0); + assert!( + matched > 0, + "{}: the needle should match some rows", + shape.name + ); + + // A range at the ordered path: the functional btree. + let operand = leaf(shape.rows - 5); + let text = explain( + &mut db, + &format!("SELECT id FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf"), + &[&number, &operand], + ); + assert!( + text.contains(&format!("Index Scan using {table}_ord")) + || text.contains(&format!("Bitmap Index Scan on {table}_ord")), + "{} on PostgreSQL {version}: a range at an ordered path should use the btree without help:\n{text}", + shape.name + ); + let matched: i64 = db + .query_one( + &format!("SELECT count(*) FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf"), + &[&number, &operand], + ) + .unwrap() + .get(0); + assert_eq!(matched, 5, "{}: the top five numbers", shape.name); + } + + // The control: with `json_tree_query` priced as if it were free, the + // planner takes the sequential scan over the TOASTed table. + let mut tx = db.transaction().unwrap(); + tx.batch_execute("ALTER FUNCTION eql_v4.json_tree_query(public.eql_v4_json_tree) COST 1") + .unwrap(); + let operand = needle(shapes[1].needle.clone()); + let text = tx + .query( + "EXPLAIN SELECT id FROM json_tree_plan_nested WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree", + &[&operand], + ) + .unwrap() + .iter() + .map(|r| r.get::<_, String>(0)) + .collect::>() + .join("\n"); + tx.rollback().unwrap(); + assert!( + text.contains("Seq Scan on json_tree_plan_nested"), + "on PostgreSQL {version}, with the function priced as free the planner should take the sequential scan, which is what the cost guards against:\n{text}" + ); +} From 1f366a7dbdcafad11bb4f5766db7df119dcd44a8 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:43:15 +1100 Subject: [PATCH 2/7] docs(eql): state what the json_tree entry order and extraction reveal The leakage page said a stored row hides its shape, its parent relations, its array lengths and its key names, and that tampering never changes what a value decrypts to. Read against the code, none of that holds. `sv` and `q` are stored in depth-first pre-order, an object's members in sorted key order, under column-deterministic selectors. The order therefore fixes each row's shape, makes the parent relation inferable (exact for top-level fields under the sealed root), counts every array, and shows the alphabetical order of sibling keys. A subtree read sends the token of the top-level field the path starts with, not of the node asked for. With the sealed root off it sends the root's token. A position gate carries its position in the clear. An extracted entry opens under the selector the server returns with it, so a row's fields can be swapped. A bound's OPE term places it exactly among the stored values, not roughly. The page now states the format as built, field by field, including the header. The v3 comparison and the sealed-root option are corrected. The design document notes where section 6 was wrong, and records the entry order and the extraction selector as open decisions for the owner. The format is unchanged. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../eql-json-tree-planner-and-leakage.md | 2 +- docs/plans/2026-10-17-json-tree-index.md | 6 ++- .../eql/docs/reference/json-tree-leakage.md | 43 +++++++++++-------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/.changeset/eql-json-tree-planner-and-leakage.md b/.changeset/eql-json-tree-planner-and-leakage.md index b08b2b923..7479de1cc 100644 --- a/.changeset/eql-json-tree-planner-and-leakage.md +++ b/.changeset/eql-json-tree-planner-and-leakage.md @@ -2,4 +2,4 @@ "@cipherstash/eql": patch --- -EQL v4 `json_tree` (unreleased): the reference docs gain "What an encrypted JSON tree reveals" (`docs/reference/json-tree-leakage.md`), one page on what a stored row shows the database at rest, what each query type hands it, what is authenticated, and the side-by-side with v3 `ste_vec`. The encryption test suite now holds the planner to the GIN and btree indexes with every setting at its default, on inline and TOASTed documents, and CI runs that suite once per supported PostgreSQL version. +EQL v4 `json_tree` (unreleased): the reference docs gain "What an encrypted JSON tree reveals" (`docs/reference/json-tree-leakage.md`), one page on what a stored row shows the database at rest (including the row's shape, which the stored depth-first entry order gives away), what each query type hands it, what is and is not authenticated, and the side-by-side with v3 `ste_vec`. The encryption test suite now holds the planner to the GIN and btree indexes with every setting at its default, on inline and TOASTed documents, for one-shot and prepared statements, and CI runs that suite on every supported PostgreSQL version. diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index 631efcc68..dc2d58018 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -278,6 +278,8 @@ On a flat five-node document the proposal is about 8% larger: the two chain fiel > **2026-10-22:** parent tags are position-keyed (Phase 1), so siblings do not share a `p` at rest. Since Phase 3b there is no `g`: which leaves sit in the same array element is visible at rest through the nesting of the stored projection `q` instead. +> **2026-10-23 (review of #1336):** the "proposed" column of the shape-equality and parent-and-child rows is wrong for the format as built, and was wrong for the design as written. Neither the tags nor their salt are what decides it: the entries of `sv` (and the pairs of `q`) are stored in depth-first pre-order, an object's members in sorted key order, under selectors that are the same in every row. A row's sequence of selectors therefore fixes its shape exactly, the parent of every node is inferable by comparing rows (and is exact for the top-level fields under the sealed root, whose entries are the ones with no `p`), every array's length is the count of its element selector, and the relative order of sibling key names is visible across rows. `packages/eql/docs/reference/json-tree-leakage.md` states the format as built; the decision this leaves open is recorded under Phase 4. + ### 7. Performance - **Encrypt:** per node two PRFs (`s`, `vs`), three hashes (`t`, `p`, mask), one small AEAD. v3 does two MACs and one AEAD over the subtree bytes. Fewer bytes encrypted at every depth above the leaves. @@ -488,7 +490,9 @@ The nested document is now about 14 times its plaintext on disk (18 at Phase 3c, Two of the three items that gate a release are closed here; the third, the final re-bench, follows them. - **Planner fragility (gate 2).** `eql_v4.json_tree_query` is plpgsql with `COST 1000` so that a bitmap scan over the GIN index beats a sequential scan even when the documents are TOASTed and the heap looks like a few pages. That was observed in the benches, at 10,000 and 100,000 rows, on one PostgreSQL version, and nothing held it. `postgres_the_planner_picks_the_indexes_without_help` in the encryption suite now does: on a flat table (1,000 four-node documents, every row inline) and a nested table (300 documents of about 220 nodes, every row TOASTed, both regimes pinned by `pg_column_size`), with every planner setting at its default and after `ANALYZE`, containment plans as a bitmap scan over the GIN index and a range at an ordered path as a scan of the functional btree on `ord_term(doc -> selector)`. A control sets the function's cost to 1 for one transaction and shows the planner taking the sequential scan on the nested table, so the assertion is known to be able to fail. The suite runs in `test-eql.yml` in a new `rust-encryption-postgres` job, a matrix over the supported PostgreSQL versions (17 on a pull request, 14 to 17 on a push, like the SQLx jobs); it was a step of the single-version `rust-crates` job. Measured locally on 14, 15, 16 and 17 before the commit. -- **Leakage (gate 3).** `packages/eql/docs/reference/json-tree-leakage.md` is the one page: what a stored row shows at rest, field by field; what each query type hands the database; what is authenticated; the side-by-side with v3 from section 6 above, brought up to date (no `g`, position-keyed `p`, ordering opt-in); and the two options that change the picture. It says what a token is and that it is column-wide and does not expire, which is the sentence a customer most needs before sending one. +- **Leakage (gate 3).** `packages/eql/docs/reference/json-tree-leakage.md` is the one page: what a stored row shows at rest, field by field; what each query type hands the database; what is authenticated; the side-by-side with v3 from section 6 above, brought up to date (no `g`, position-keyed `p`, ordering opt-in) and corrected where section 6 was wrong (the entry order; see the note under its table); and the two options that change the picture. It says what a token is and that it is column-wide and does not expire, which is the sentence a customer most needs before sending one. +- **Open for the owner: the stored entry order (2026-10-23, review of #1336).** `sv` and `q` hold the nodes in the walk's order, depth-first, an object's members in the sorted order of their keys (serde_json's `Map` without `preserve_order`; a build that unifies that feature on would store insertion order instead, which leaks the application's field order). Across rows this reveals each row's shape, its parent relation, every array's length and the alphabetical order of sibling key names, which partly undoes what the keyed rank (decision 11) hides from a token holder. The leakage page says so. Closing it is a format change, not a documentation one: a stored order that does not follow the tree (a document-keyed permutation, say, with `j` the permuted position), a rebuild that does not read the tree from the order, and the same for the pairs of `q`, whose nesting would still group array elements. Not taken here. +- **Open for the owner: an extracted entry opens under the selector the server returns with it.** `JsonNode` and `JsonSubtree` open each entry under the `s` and `j` that arrive beside it, so a decrypted value is always one sealed in that row but not necessarily the one at the path asked for (`->` on `$.salary` can be answered with the row's `$.bonus`, or with the sealed root, which opens to the whole document). The leakage page states this under "What is authenticated". Checking the returned `s` against the selector the client derived would close it for `->`; a walk's rows would need each child's selector checked against its parent's path. ## Decisions to take before Phase 1 diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 9c87b8f15..06953a539 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -1,6 +1,6 @@ # What an encrypted JSON tree reveals -`public.eql_v4_json_tree` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page reveals one. What the database can see is a bounded amount of **structure**, and it is deliberate: each item below is what one query capability costs. Read this before you put a column on it, and decide the two options at the end with it in mind. +`public.eql_v4_json_tree` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page shows the text of one. What the database can see is **structure**, and a great deal of it: which paths a row has and how many times, the order the nodes are stored in, which (path, value) pairs repeat across rows, and lengths. Each item below is what one query capability or one storage choice costs. Read this before you put a column on it, and decide the two options at the end with it in mind. ## On this page @@ -12,30 +12,34 @@ ## What a stored row shows -Someone who holds the column and no key, such as a database administrator, a backup, or an attacker with a copy of the table, learns the following from each row. "Across rows" means a comparison between rows of the same column; nothing links a column to another column or another keyset. +Someone who holds the column and no key, such as a database administrator, a backup, or an attacker with a copy of the table, learns the following from each row. "Across rows" means a comparison between rows of the same column. No selector, value selector or ordering term links one column to another; the header names the keyset, which is the same for every column it seals. | field | what it reveals | |---|---| -| `s`, the path selector on every node | **Which paths the row has.** A selector is a keyed hash of the path, the same in every row of the column, so two rows with a selector in common hold the same path. Key names are inside the hash and not readable. Array elements share their array's selector, so `$.tags[0]` and `$.tags[1]` look alike. | -| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value. Not the value itself, and not whether two different paths hold the same value. **Which nodes sit inside the same array element**, through the nesting. That grouping is what makes containment exact for arrays of objects, and it is the one thing this format reveals that v3 did not. | -| the number of entries, and each ciphertext's length | **The node count**, and for each scalar the length of its value plus its key name, to within the AEAD's fixed overhead. A row's total size is the usual size signal any encrypted column gives. | -| `p`, the parent tag below the chain's heads | Nothing on its own. It is salted by the document's `iv`, so the same position in two rows is unrelated bytes. It shows **how many nodes are direct members of the root** (they carry no `p`); not which node is any other node's parent, not an array's length, and not whether two rows have the same shape. | -| `op`, the ordering term, only on leaves at the paths the index orders | **The order of that leaf's value among every row that holds one.** The term is order-preserving encryption under one key per column, so leaves at two ordered paths compare with each other too. An order-preserving term is not a random value: values that sort near each other get terms that share leading bytes. By default no path is ordered; see the options below. | +| `s`, the path selector on every node | **Which paths the row has, and how many nodes sit at each.** A selector is a keyed hash of the path, the same in every row of the column, so two rows with a selector in common hold the same path. Key names are inside the hash and not readable. Array elements share their array's selector, so `$.tags[0]` and `$.tags[1]` look alike, but each element is its own entry: the number of entries carrying `$.tags[*]`'s selector is the length of `$.tags` (the total over every array at that path, when the path runs through another array). | +| the order of the entries in `sv` | **The row's shape.** Entries are stored in depth-first order, the root first, each node followed by everything beneath it, and an object's members in the sorted order of their key names. So a row's sequence of selectors fixes its shape exactly: two rows with the same sequence have the same tree with the same key names, and two rows that differ show where. Which selector's path sits under which is not written down, but it is the same in every row, so an observer comparing rows learns it, and with it each row's parent and child relations and the node count of every subtree. Siblings stored in sorted key order also show, across rows, **the alphabetical order of sibling key names** (for example that the key behind one selector sorts before the key behind another), which narrows guesses at what each selector names. | +| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element, in the same order | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value (for an object or an array, of its kind alone). So at each path the database sees which rows agree and how often each value occurs: a frequency histogram of every path, and whether one row's document contains another's. Not the value itself, and not whether two different paths hold the same value. **Which nodes sit inside the same array element**, through the nesting, for every element that is an object or an array. That grouping is what makes containment exact for arrays of objects, and it is the one thing this format reveals that v3 did not. | +| `p`, the parent tag below the chain's heads | Nothing beyond the entry order on its own. It is salted by the document's `iv`, so the same position in two rows is unrelated bytes, and it does not say which entry is the parent. Which entries carry no `p` does show: with the sealed root on, those are the root's direct members, and since everything beneath a member follows it, **the node count of each top-level field's subtree** can be read off a single row exactly. With the sealed root off, only the root lacks one. | +| each ciphertext's length, and the number of entries | **The node count**, and for each node the length of its key name (for an object member) plus, for a scalar, the length of its value, exactly, less the AEAD's fixed overhead. A string's length in bytes and a number's length in digits are therefore visible per node. With the sealed root on, the root entry's length is the document's length as JSON. | +| `op`, the ordering term, only on number and string leaves at the paths the index orders | **The order of that leaf's value among every row that holds one, and which rows hold equal values.** The term is order-preserving encryption (CLLW OPE) under one key per column, so leaves at two ordered paths compare with each other too. It is deterministic and not a random value: two values whose encodings share leading bytes (for a string, its first characters) get terms that share leading bytes, so the term also shows how long that common prefix is. A term's presence at an ordered path also says the leaf is a number or a string. By default no path is ordered; see the options below. | +| `h`, the header | The keyset the data key was minted under, the data key's identifier (unique to the row), and whether the root is sealed. `v` and `i` are the version and the table and column identifier every EQL value carries. | -Not revealed at rest: any value, any key name, which node is whose parent, how long an array is, and whether two rows have the same shape. +Not revealed at rest: any value's or key name's text, and whether two different paths hold the same value. Revealed at rest, by the entry order, although no field stores it: the shape of every row, the parent of every node, every array's length, and the sorted order of sibling key names. ## What a query shows A query hands the database a term derived from the plaintext it asks about, and the database's answer is visible to it. Each query type reveals the following, in addition to the rows it returns. -- **Containment (`@>`, `<@`).** The needle's own projection: the selectors of its paths, the value selectors of its leaves, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. -- **Path extraction (`->`, `->>`, `json_tree_path_query`).** The selector of the path asked. The entry returned is a ciphertext. -- **Ordering at a path (`<`, `<=`, `>`, `>=`, `ORDER BY ord_term(…)`).** The bound's own ordering term, so roughly where the bound sits in the order of the stored values at that path. -- **A subtree read (`json_tree_subtree`, `json_tree_subtree_at`, `json_tree_children`, `json_tree_child_count`).** A **token** for the node, and one gate (a selector or an array position) per step below it. A token is a capability over the structure of that subtree in *every row of the column*: whoever holds it can count the nodes under that path in any row, see how they nest and in what order, and derive the tokens of the nodes beneath. It still opens no value. Tokens are column-wide and do not expire, so a token sent once is useful to the database for as long as the column lives. Send a token to the database in a query and nowhere else. With the sealed root on (the default) the broadest token a query can send covers one top-level field's subtree; no query can walk a whole column. +- **Containment (`@>`, `<@`), and equality at a path, which is sent as a containment needle.** The needle's own projection: the selectors of its paths, the value selectors of its nodes, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. +- **Path extraction (`->`, `->>`, `json_tree_path_query`, `json_tree_path_exists`).** The selector of the path asked. A position in the path reads as the wildcard, so a position is not sent. The entry returned is a ciphertext. +- **Ordering at a path (`<`, `<=`, `>`, `>=`).** The path's selector and the bound's own ordering term, which compares with the stored terms exactly as they compare with each other: the database learns the bound's place in the order of the stored values at every ordered path, and whether it equals one. `ORDER BY ord_term(…)` sends no term and shows nothing the stored terms do not. +- **A subtree read (`json_tree_subtree`, `json_tree_subtree_at`, `json_tree_children`, `json_tree_child_count`).** A **token**, and one gate per step below it. The token is not the token of the node you asked for: it is the token of the deepest node on the path the client can derive, which is the top-level field the path starts with or, while the path goes on through array positions, the element they reach. Reading `$.profile.address.city` sends `$.profile`'s token. A gate is the step's selector or, for an array position, `#n:`, with the position `n` in the clear. A token is a capability over the structure of its node's subtree in *every row of the column*: whoever holds it can find each node under that field in any row, see how the nodes nest, link each one to its parent, and derive the tokens of the nodes beneath. It opens no value. Tokens are column-wide and do not expire, so a token sent once is useful to the database for as long as the column lives. Send a token to the database in a query and nowhere else. With the sealed root on (the default) the broadest token a query can send covers one top-level field's subtree, and no token covers a whole document. With it off, every subtree read whose path begins with a key sends the root's token, which covers every row's whole document. ## What is authenticated -Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under this data key, at this path (`s`) and at this position in the document (`j`); a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the ordering terms, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). The consequence to design around: tampering with those fields changes **which rows a query returns or where a walk stops**, never what a value decrypts to. A query's result set is not integrity-protected; a decrypted value is. +Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under this row's data key, with the selector `s` and the position `j` it is opened with; a ciphertext moved to another row or column, or opened under another selector or position, fails. When the client opens a whole document it takes each entry's position from its place in `sv`, so a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the ordering terms, the header's sealed-root flag, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). + +The consequence to design around: tampering with those fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the client opens it under those, so a decrypted value is always one that was sealed in that row, under that key, but the client does not check today that it is the entry at the path it asked for: a database that wants to can answer `$.salary` with the row's `$.bonus`. A query's result is not integrity-protected; each decrypted value is genuine, and comes from the row the result says. ## Compared with EQL v3 (`public.eql_v3_json_search`) @@ -44,17 +48,20 @@ Only the ciphertext `c`. Opening an entry verifies that it was sealed under this | which paths a row has | yes | yes | | which (path, value) pairs a row has | yes | yes | | node count | yes, two entries per node | yes, one entry per node | -| size of every subtree | yes: each path entry's ciphertext is the subtree under it | no: only each scalar's length | -| parent and child relation | inferable from subtree sizes and co-occurrence across rows | hidden at rest; linkable only with a token the query supplies | -| whether two rows have the same shape | yes, selectors are column-deterministic | no, until a token is supplied | +| array lengths | yes | yes, from how many entries carry the element selector | +| size of every subtree | yes: each path entry's ciphertext is the subtree under it | no ciphertext spans a subtree, but its node count follows from the entry order, and its size is the sum of its entries' lengths | +| parent and child relation | inferable from subtree sizes and co-occurrence across rows | not stored, but inferable from the entry order and co-occurrence across rows; exact for top-level fields under the sealed root | +| whether two rows have the same shape | yes, selectors are column-deterministic | yes, selectors are column-deterministic and entries are stored in depth-first order | | which leaves sit in the same array element | no | yes, through the projection's nesting: the one new disclosure, required for exact containment, bounded to array elements | | equal values in sibling elements | ciphertexts equal | ciphertexts differ | | ordering terms | on scalar leaves, with no per-path choice | only at the paths the index orders; none by default | -| a subtree read | reveals the selector asked | reveals the token asked, and with it the subtree's shape in every row for the life of the column | +| a subtree read | reveals the selector asked | reveals a token for the top-level field the path starts with, and with it that field's shape in every row for the life of the column | ## The two options that change the picture -- **`sealed_root`** (default on). The root entry's ciphertext is the whole document, so a full read is one decryption and the broadest token a query can send is one top-level field's. The cost is that the root entry's length reveals the document's size, and the row is larger. +- **`sealed_root`** (default on). The root entry's ciphertext is the whole document, so a full read is one decryption and the broadest token a query can send is one top-level field's. The costs are that the root entry's length reveals the document's size, the row is larger, and the root's direct members are marked by their missing `p`, which puts each top-level field's node count in plain view. Off, there is no document-sized ciphertext, but every subtree read whose path begins with a key sends the root's token, the whole column's shape. - **`ordered`** (default none). Each path listed carries an ordering term and takes on the `op` row above; a path not listed cannot be range-queried or sorted, and a query that tries is refused by the client before any key loads. Order only the paths your queries sort or range over. +Two disclosures above are properties of the stored format that no option changes today: the depth-first entry order, with members in sorted key order, and the client opening an extracted entry under the selector the database returns with it. Both are open decisions recorded in the design document (`docs/plans/2026-10-17-json-tree-index.md` at the repository root, Phase 4 notes). + To report a concern with any of this, see `SECURITY.md` at the repository root. From d01d5471e1ef59aec9bf733190a5208cf890b8e9 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:43:15 +1100 Subject: [PATCH 3/7] ci(eql): gate ci-required on the per-version encryption suite Moving the PostgreSQL half of the Rust encryption suite out of `rust-crates` into `rust-encryption-postgres` also took it out of `ci-required`. That job reads only the results of the jobs it needs, so a failing oracle or plan-shape test would have left the aggregate check green. The job is now listed. eql-suite-ci.test.mjs fails when any job in the workflow is missing from `ci-required`'s needs, which the old list would have tripped. The `validate` job's comment, which the new job had been inserted under, moves back above `validate`. The paths comment now names both jobs that compile the stack-* crates by path. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .github/workflows/test-eql.yml | 18 +++++++++------- scripts/__tests__/eql-suite-ci.test.mjs | 28 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index 35034b7b6..438cfb3bf 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -100,7 +100,8 @@ on: - "packages/eql/docker/**" - "packages/eql/README.md" - "packages/eql/SUPABASE.md" - # The `rust-crates` job's encryption steps compile these by PATH + # The `rust-crates` and `rust-encryption-postgres` jobs' encryption + # steps compile these by PATH # (`eql-bindings`' `stack-encrypt` feature): stack-encrypt, its derive # and stack-kms through `path =`, and stack-auth and stack-profile # through `workspace = true` entries the root `Cargo.toml` resolves to @@ -449,9 +450,6 @@ jobs: run: | mise run test:sqlx:partition - # docs:validate + Clean-DB v3 install smoke. Both are version-relevant, so - # they follow the event's PG set (PG17 on PR; 14-17 in the queue). Moved out - # of the old per-version test job so they run ONCE per version, not per shard. # The Rust encryption suite against a live database, once per supported # PostgreSQL version: the JSON tree family's oracle (containment over the # encrypted column against native `jsonb @>` over the plaintexts), its @@ -459,7 +457,9 @@ jobs: # the GIN and btree indexes with every setting at its default. Plan choice # is a per-version property of the planner, which is why this is a matrix # and not a step of `rust-crates`. PR and manual dispatch run PG 17 alone; - # a push runs the full set, like the SQLx jobs (see `setup`). + # a push or the schedule runs the full set, like the SQLx jobs (see + # `setup`). Listed in `ci-required`'s `needs`, like every job here: the + # step it replaced sat inside `rust-crates`, which is. rust-encryption-postgres: name: "Rust encryption with PostgreSQL ${{ matrix.postgres-version }}" needs: [changes, setup] @@ -514,6 +514,9 @@ jobs: EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.encryption-postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres" run: mise run test:encryption:postgres + # docs:validate + Clean-DB v3 install smoke. Both are version-relevant, so + # they follow the event's PG set (PG17 on PR; 14-17 in the queue). Moved out + # of the old per-version test job so they run ONCE per version, not per shard. validate: name: "Validate (Postgres ${{ matrix.postgres-version }})" needs: [changes, setup] @@ -1053,8 +1056,9 @@ jobs: ci-required: name: "ci-required" needs: [changes, setup, build-archive, test, validate, schema, rust-crates, - codegen, self-contained-v3, matrix-coverage, splinter, docs-static, - known-failures, doc-anchors, e2e] + rust-encryption-postgres, codegen, self-contained-v3, + matrix-coverage, splinter, docs-static, known-failures, + doc-anchors, e2e] if: always() runs-on: blacksmith-16vcpu-ubuntu-2204 # Same opt-out as `setup`, and this is the worse of the two to get wrong: diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index ee00032a8..1c9dc60fe 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -200,6 +200,34 @@ describe('a root workflow runs the EQL SQLx suite', () => { }) }) +describe('`ci-required` aggregates every job in the EQL workflow', () => { + // `ci-required` is the one check meant to gate a merge, and it reads + // `needs.*.result`, so a job missing from its `needs:` can fail without + // turning it red. Silent when wrong: the failing job still shows in the + // checks list, and the aggregate beside it is green. The PostgreSQL half of + // the Rust encryption suite left `rust-crates` (which is listed) for a job + // of its own and was missing from the list until the review of #1336. + const AGGREGATOR = 'ci-required' + + it('lists every other job in its `needs:`', () => { + const jobs = readWorkflow(EQL_WORKFLOW)?.jobs ?? {} + const names = Object.keys(jobs) + // Guards the guard: an empty job map passes the comparison below. + expect(names.length).toBeGreaterThan(1) + expect(names, `${EQL_WORKFLOW} has no \`${AGGREGATOR}\` job`).toContain( + AGGREGATOR, + ) + const needs = [jobs[AGGREGATOR]?.needs ?? []].flat() + const missing = names.filter( + (name) => name !== AGGREGATOR && !needs.includes(name), + ) + expect( + missing, + `These jobs in ${EQL_WORKFLOW} are not in \`${AGGREGATOR}\`'s \`needs:\`, so a failure in them leaves the required check green:\n${missing.map((name) => ` ${name}`).join('\n')}`, + ).toEqual([]) + }) +}) + describe('the relevance filter still selects the imported tree', () => { const wf = readWorkflow(EQL_WORKFLOW) From 491b09c8111e4bc0863fbd6a5d47dcbe792a9aaa Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:43:15 +1100 Subject: [PATCH 4/7] test(eql): pin TOAST storage exactly and hold prepared-statement plans The planner test inferred "every row TOASTed" from pg_column_size, which measures a value's stored size, not where it is stored. It now counts the distinct chunk ids in the table's TOAST relation. That works on PostgreSQL 14 to 17, where pg_column_toast_chunk_id is 17 only. It also asked only one-shot statements, which are planned with their operand in view. A pooled driver prepares once and executes many times, and after five custom plans `plan_cache_mode = auto` may plan generically, with the operand as a parameter. Containment and the range at an ordered path are now also asserted from a prepared statement past its fifth execution, with EXPLAIN EXECUTE showing the plan the cache would run. Verified on PostgreSQL 14, 15, 16 and 17. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../eql/tests/encryption/tests/json_tree.rs | 83 ++++++++++++++++--- 1 file changed, 73 insertions(+), 10 deletions(-) diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 1fef27654..202129658 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -911,7 +911,10 @@ fn nested_document(i: i64) -> Value { /// The oracle test above shows the GIN index with sequential scans turned /// off; this one shows it with them on, which is what a customer's /// database runs. The ordering path is held the same way: a functional -/// btree on `ord_term(doc -> selector)` serves a range at a path. +/// btree on `ord_term(doc -> selector)` serves a range at a path. Both +/// are asked twice: as one-shot statements, planned with their operands +/// in view, and as a prepared statement executed past its fifth run, +/// where `plan_cache_mode = auto` may plan once for any operand. /// /// The last block is a control: with the function's cost set to the /// minimum for one transaction, the planner prefers the sequential scan @@ -1016,19 +1019,35 @@ fn postgres_the_planner_picks_the_indexes_without_help() { db.batch_execute(&format!("ANALYZE {table}")).unwrap(); // The two storage regimes the shapes stand for, pinned, so the test - // keeps meaning when a document's size changes. + // keeps meaning when a document's size changes. Read from the TOAST + // relation itself: a value stored out of line is one `chunk_id` + // there, so the count of distinct chunk ids is the count of + // TOASTed rows, on every supported version (`pg_column_toast_chunk_id` + // is PostgreSQL 17 only, and `pg_column_size` reports the stored + // size, which says nothing about where it is stored). let (column, toasted): (i32, i64) = { - let row = db + let column: i32 = db .query_one( - &format!( - "SELECT avg(pg_column_size(doc))::int, - (SELECT count(*) FROM {table} WHERE pg_column_size(doc) > 2000) - FROM {table}" - ), + &format!("SELECT avg(pg_column_size(doc))::int FROM {table}"), &[], ) - .unwrap(); - (row.get(0), row.get(1)) + .unwrap() + .get(0); + let toast: String = db + .query_one( + "SELECT reltoastrelid::regclass::text FROM pg_class WHERE oid = $1::text::regclass", + &[&table], + ) + .unwrap() + .get(0); + let toasted: i64 = db + .query_one( + &format!("SELECT count(DISTINCT chunk_id) FROM {toast}"), + &[], + ) + .unwrap() + .get(0); + (column, toasted) }; if shape.inline { assert_eq!( @@ -1090,6 +1109,50 @@ fn postgres_the_planner_picks_the_indexes_without_help() { .unwrap() .get(0); assert_eq!(matched, 5, "{}: the top five numbers", shape.name); + + // The same two queries as a pooled driver runs them: one prepared + // statement, executed past the five custom plans after which + // `plan_cache_mode = auto` may switch to a generic plan, whose + // operand is a parameter the planner cannot see. The selector stays + // a literal, as it has to for the functional index to match. + // `EXPLAIN EXECUTE` shows the plan the cache would run next. + let containment = needle(shape.needle.clone()); + let range = leaf(shape.rows - 5); + for literal in [&containment, &range] { + assert!(!literal.contains('\''), "an operand embeds as a literal"); + } + db.batch_execute(&format!( + "PREPARE {table}_contains(text) AS + SELECT id FROM {table} WHERE doc @> $1::jsonb::eql_v4.query_json_tree; + PREPARE {table}_range(text) AS + SELECT id FROM {table} WHERE (doc -> '{number}'::text) >= $1::jsonb::eql_v4.query_json_tree_leaf;" + )) + .unwrap(); + for (statement, operand, index, what) in [ + ("contains", &containment, "_gin", "containment"), + ("range", &range, "_ord", "a range at an ordered path"), + ] { + for _ in 0..6 { + db.batch_execute(&format!("EXECUTE {table}_{statement}('{operand}')")) + .unwrap(); + } + let text = db + .simple_query(&format!("EXPLAIN EXECUTE {table}_{statement}('{operand}')")) + .unwrap() + .into_iter() + .filter_map(|message| match message { + postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), + _ => None, + }) + .collect::>() + .join("\n"); + assert!( + text.contains(&format!("Index Scan using {table}{index}")) + || text.contains(&format!("Bitmap Index Scan on {table}{index}")), + "{} on PostgreSQL {version}: {what} from a prepared statement past its fifth execution should use the index:\n{text}", + shape.name + ); + } } // The control: with `json_tree_query` priced as if it were free, the From 17ac84fa6a2a068198164686468055d83e6529c7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:44:48 +1100 Subject: [PATCH 5/7] docs(eql): do not claim an extracted value comes from the row it is listed under An extracted entry carries the header of the document it was sealed in, and the client opens it under that header. A database can therefore list another row's entry, from the same column, in a row's place, and it opens. The page said a decrypted value "comes from the row the result says". It now says the value is genuine and from this column, and that both its path and its row are the database's word. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/eql/docs/reference/json-tree-leakage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 06953a539..77e9eb852 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -39,7 +39,7 @@ A query hands the database a term derived from the plaintext it asks about, and Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under this row's data key, with the selector `s` and the position `j` it is opened with; a ciphertext moved to another row or column, or opened under another selector or position, fails. When the client opens a whole document it takes each entry's position from its place in `sv`, so a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the ordering terms, the header's sealed-root flag, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). -The consequence to design around: tampering with those fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the client opens it under those, so a decrypted value is always one that was sealed in that row, under that key, but the client does not check today that it is the entry at the path it asked for: a database that wants to can answer `$.salary` with the row's `$.bonus`. A query's result is not integrity-protected; each decrypted value is genuine, and comes from the row the result says. +The consequence to design around: tampering with those fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the client opens it under those, and the header it carries names the data key, so a decrypted value is always one that was sealed in this column, but the client does not check today that it is the entry at the path it asked for, or that it comes from the row the result puts it in: a database that wants to can answer `$.salary` with the same row's `$.bonus`, or with another row's `$.salary`. A query's result is not integrity-protected; each decrypted value is genuine, but where it came from is the database's word. ## Compared with EQL v3 (`public.eql_v3_json_search`) From b8d86cd8d7e1412d318aa4337e29d437fd86b0fc Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 24 Oct 2026 14:27:59 +1100 Subject: [PATCH 6/7] docs(eql): say an extracted entry proves its column, not its row The Phase 4 note said a decrypted value is always one sealed in that row. It is not: the entry's AAD binds the keyset, the column's context, the data key's tag, the selector and the ordinal, and nothing that names a row. An entry the database returns together with its own row's header opens, so `->` can be answered with another row's value at the same path. That is the cross-row interchangeability ADR 0010 accepts for every value sealed under one context, so the fix is to the wording. The note now says what is bound, which substitutions open, and that a selector check would close the path substitution but not the row one. The leakage page's "What is authenticated" paragraph said "under this row's data key"; it now says the key the row's header names and points at the paragraph below it, which already described the row case. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- docs/plans/2026-10-17-json-tree-index.md | 2 +- packages/eql/docs/reference/json-tree-leakage.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index 409c2f220..71a4e3cb3 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -492,7 +492,7 @@ Two of the three items that gate a release are closed here; the third, the final - **Planner fragility (gate 2).** `eql_v4.json_tree_query` is plpgsql with `COST 1000` so that a bitmap scan over the GIN index beats a sequential scan even when the documents are TOASTed and the heap looks like a few pages. That was observed in the benches, at 10,000 and 100,000 rows, on one PostgreSQL version, and nothing held it. `postgres_the_planner_picks_the_indexes_without_help` in the encryption suite now does: on a flat table (1,000 four-node documents, every row inline) and a nested table (300 documents of about 220 nodes, every row TOASTed, both regimes pinned by `pg_column_size`), with every planner setting at its default and after `ANALYZE`, containment plans as a bitmap scan over the GIN index and a range at an ordered path as a scan of the functional btree on `ord_term(doc -> selector)`. A control sets the function's cost to 1 for one transaction and shows the planner taking the sequential scan on the nested table, so the assertion is known to be able to fail. The suite runs in `test-eql.yml` in a new `rust-encryption-postgres` job, a matrix over the supported PostgreSQL versions (17 on a pull request, 14 to 17 on a push, like the SQLx jobs); it was a step of the single-version `rust-crates` job. Measured locally on 14, 15, 16 and 17 before the commit. - **Leakage (gate 3).** `packages/eql/docs/reference/json-tree-leakage.md` is the one page: what a stored row shows at rest, field by field; what each query type hands the database; what is authenticated; the side-by-side with v3 from section 6 above, brought up to date (no `g`, position-keyed `p`, ordering opt-in) and corrected where section 6 was wrong (the entry order; see the note under its table); and the two options that change the picture. It says what a token is and that it is column-wide and does not expire, which is the sentence a customer most needs before sending one. - **Open for the owner: the stored entry order (2026-10-23, review of #1336).** `sv` and `q` hold the nodes in the walk's order, depth-first, an object's members in the sorted order of their keys (serde_json's `Map` without `preserve_order`; a build that unifies that feature on would store insertion order instead, which leaks the application's field order). Across rows this reveals each row's shape, its parent relation, every array's length and the alphabetical order of sibling key names, which partly undoes what the keyed rank (decision 11) hides from a token holder. The leakage page says so. Closing it is a format change, not a documentation one: a stored order that does not follow the tree (a document-keyed permutation, say, with `j` the permuted position), a rebuild that does not read the tree from the order, and the same for the pairs of `q`, whose nesting would still group array elements. Not taken here. -- **Open for the owner: an extracted entry opens under the selector the server returns with it.** `JsonNode` and `JsonSubtree` open each entry under the `s` and `j` that arrive beside it, so a decrypted value is always one sealed in that row but not necessarily the one at the path asked for (`->` on `$.salary` can be answered with the row's `$.bonus`, or with the sealed root, which opens to the whole document). The leakage page states this under "What is authenticated". Checking the returned `s` against the selector the client derived would close it for `->`; a walk's rows would need each child's selector checked against its parent's path. +- **Open for the owner: an extracted entry opens under the selector the server returns with it.** `JsonNode` and `JsonSubtree` open each entry under the `s` and `j` that arrive beside it, and under the data key the header that arrives with it names. The entry's AAD (`entry_aad` in `packages/stack-encrypt/src/json/derive.rs`) binds the keyset, the column's context, the data key's tag, the selector and the ordinal, and nothing that identifies a row. So a decrypted value is always one sealed in that column, but not necessarily the one at the path asked for, nor one from the row the result puts it in: `->` on `$.salary` can be answered with the row's `$.bonus`, with the sealed root (which opens to the whole document), or with another row's `$.salary` returned together with that row's header. The last is by design: two values sealed under the same context are interchangeable between rows (ADR 0010 in `packages/stack-encrypt/docs/adr`), and row provenance is the database's word. The leakage page states this under "What is authenticated". Checking the returned `s` against the selector the client derived would close the path substitution for `->`, though not the row one; a walk's rows would need each child's selector checked against its parent's path. ## Decisions to take before Phase 1 diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 77e9eb852..bfb6ffab2 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -37,7 +37,7 @@ A query hands the database a term derived from the plaintext it asks about, and ## What is authenticated -Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under this row's data key, with the selector `s` and the position `j` it is opened with; a ciphertext moved to another row or column, or opened under another selector or position, fails. When the client opens a whole document it takes each entry's position from its place in `sv`, so a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the ordering terms, the header's sealed-root flag, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). +Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under the data key its row's header names (a fresh one for every row), with the selector `s` and the position `j` it is opened with; a ciphertext moved into another row or column, or opened under another selector or position, fails. Nothing in it names a row, so an entry that travels with its own row's header still opens; see below. When the client opens a whole document it takes each entry's position from its place in `sv`, so a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the ordering terms, the header's sealed-root flag, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). The consequence to design around: tampering with those fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the client opens it under those, and the header it carries names the data key, so a decrypted value is always one that was sealed in this column, but the client does not check today that it is the entry at the path it asked for, or that it comes from the row the result puts it in: a database that wants to can answer `$.salary` with the same row's `$.bonus`, or with another row's `$.salary`. A query's result is not integrity-protected; each decrypted value is genuine, but where it came from is the database's word. From c31b0f39bd2bd0c710ae752bf2dc568ab4d62309 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 24 Oct 2026 14:30:33 +1100 Subject: [PATCH 7/7] test(eql): hold the json_tree planner test to the generic plan too The prepared-statement check ran six EXECUTEs and read EXPLAIN EXECUTE under plan_cache_mode = auto. auto keeps a custom plan whenever it costs less than the generic one, so for the nested table the cache could go on showing a custom plan, and a generic plan that fell back to a sequential scan would fail no test. A pooled driver can reach that generic plan. After the auto assertion, each statement (containment and the range at an ordered path, on both tables) is explained again under force_generic_plan, asserted to carry its operand as $1 (so the plan read is the generic one) and to use the same index, and the setting is reset. v4_range.rs does the same for a range domain. Passes on PostgreSQL 17 (the whole test:encryption:postgres task) and 14 (this test). Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../eql/tests/encryption/tests/json_tree.rs | 58 ++++++++++++++----- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 02063956b..a60c2a9af 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -912,9 +912,12 @@ fn nested_document(i: i64) -> Value { /// off; this one shows it with them on, which is what a customer's /// database runs. The ordering path is held the same way: a functional /// btree on `ord_term(doc -> selector)` serves a range at a path. Both -/// are asked twice: as one-shot statements, planned with their operands -/// in view, and as a prepared statement executed past its fifth run, -/// where `plan_cache_mode = auto` may plan once for any operand. +/// are asked three times: as one-shot statements, planned with their +/// operands in view; as a prepared statement executed past its fifth run, +/// where `plan_cache_mode = auto` may plan once for any operand; and as +/// the same statement under `plan_cache_mode = force_generic_plan`, since +/// `auto` keeps a custom plan whenever it costs less than the generic one +/// and would otherwise leave the generic plan untested. /// /// The last block is a control: with the function's cost set to the /// minimum for one transaction, the planner prefers the sequential scan @@ -1115,7 +1118,11 @@ fn postgres_the_planner_picks_the_indexes_without_help() { // `plan_cache_mode = auto` may switch to a generic plan, whose // operand is a parameter the planner cannot see. The selector stays // a literal, as it has to for the functional index to match. - // `EXPLAIN EXECUTE` shows the plan the cache would run next. + // `EXPLAIN EXECUTE` shows the plan the cache would run next. Under + // `auto` that may still be a custom plan (the cache keeps one while + // it costs less than the generic plan), so each statement is then + // explained again under `force_generic_plan`, which is the plan a + // pooled driver can reach whatever the costs say. let containment = needle(shape.needle.clone()); let range = leaf(shape.rows - 5); for literal in [&containment, &range] { @@ -1136,22 +1143,41 @@ fn postgres_the_planner_picks_the_indexes_without_help() { db.batch_execute(&format!("EXECUTE {table}_{statement}('{operand}')")) .unwrap(); } - let text = db - .simple_query(&format!("EXPLAIN EXECUTE {table}_{statement}('{operand}')")) - .unwrap() - .into_iter() - .filter_map(|message| match message { - postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), - _ => None, - }) - .collect::>() - .join("\n"); - assert!( + let explain_execute = |db: &mut postgres::Client| { + db.simple_query(&format!("EXPLAIN EXECUTE {table}_{statement}('{operand}')")) + .unwrap() + .into_iter() + .filter_map(|message| match message { + postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), + _ => None, + }) + .collect::>() + .join("\n") + }; + let uses_index = |text: &str| { text.contains(&format!("Index Scan using {table}{index}")) - || text.contains(&format!("Bitmap Index Scan on {table}{index}")), + || text.contains(&format!("Bitmap Index Scan on {table}{index}")) + }; + let text = explain_execute(&mut db); + assert!( + uses_index(&text), "{} on PostgreSQL {version}: {what} from a prepared statement past its fifth execution should use the index:\n{text}", shape.name ); + db.batch_execute("SET plan_cache_mode = force_generic_plan") + .unwrap(); + let text = explain_execute(&mut db); + db.batch_execute("RESET plan_cache_mode").unwrap(); + assert!( + text.contains("$1"), + "{} on PostgreSQL {version}: {what} under force_generic_plan should be planned with its operand as a parameter:\n{text}", + shape.name + ); + assert!( + uses_index(&text), + "{} on PostgreSQL {version}: {what} from a prepared statement's generic plan should use the index:\n{text}", + shape.name + ); } }