diff --git a/.changeset/fix-basic-index-nullish-build.md b/.changeset/fix-basic-index-nullish-build.md new file mode 100644 index 0000000000..e5553a1c33 --- /dev/null +++ b/.changeset/fix-basic-index-nullish-build.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Keep `undefined` in the configured nulls-first position when BasicIndex builds from existing rows. Preserve indexed key ownership and public query order across mixed and prefixed values. Make Duration equality follow its fields; order indexed ZonedDateTimes by instant, PlainDates by calendar, and time-only Durations by elapsed time; and preserve required errors for mixed or unordered Temporal values. Keep custom index callback inputs consistent across index types and ignore a manual index whose order conflicts with a limited public read. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index ab02163f6e..f98064768e 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -83,6 +83,7 @@ that test identifiers must copy production's private data structures. | Offline execution | Complete | FIFO retry, scheduler eligibility, leadership replay, transaction settlement, and typed wire serialization are literate. The settlement oracle's generated FIFO histories draw each transaction as one record (outcome; manual commit, auto-commit with the option set or omitted, or an `OfflineAction`; receipt read or not), use `isPersisted` as an auto-commit's receipt, and require that no failure escapes as an unhandled rejection, also when nobody reads the receipt, and that settlement logs no failure again and writes no debug output. An empty transaction on any commit path may precede each one; its receipt and completion waiter must fulfil without reaching the provider or the outbox. A manual transaction that writes a row and rolls back before it commits may also precede each one; its waiter must reject with an Error that names the rollback. A focused witness rolls a transaction back while it persists, for both provider outcomes: its waiter must wait for the executor and report the executor's outcome. After every transaction settles, a new waiter for each one must reject within a turn rather than hang. An unread receipt still settles with its own outcome when read afterwards, and a caller may register its completion waiter before admission on every path but `OfflineAction`. A focused witness fails the admission outbox write on every commit path and requires the receipt to reject with the storage error, without an unhandled rejection or a repeated log; a failed outbox write stops the executor, so generated histories do not continue past one. Open: the generated failure source is the mutation function only; manual commits always follow their own `mutate()` at once, so a pending manual transaction behind a failing shared-key sibling is not generated; a throwing `mutate()` callback is not generated, because core keeps its transaction pending; a rolled-back-before-commit transaction is generated only on the manual path, with its own key and its waiter registered after `mutate()`, so a secondary rollback of a pending transaction by a failing sibling, whose waiter gets the generic rollback Error, is not generated. | | Frameworks | Complete | Shared live-query and infinite-query models are literate. Each framework keeps its own realm, ownership, and scheduling driver. | | Structural values and ordered primitives | Complete | Structural hashing, deep equality, comparison, cursor denotation, index refinement, and query-identity output equivalence are literate. | +| Mixed-kind ordered prefixes | Bounded | Eager BasicIndex and BTreeIndex paths compare ascending and descending direct and live-query windows with an unindexed scan across mixed value kinds and paired source changes. One retained on-demand BTree history checks settled windows after a number/bigint transition, a finite request for a missing row, and a cursor request after index eligibility changes. Other provider paging histories remain open. | | Boundary refinements | Complete | Cleanup/restart admission, metadata publication, retained state, acquisition cells, D2 source reconciliation, top-K support windows, and nested Query work bounds are literate. | | Small structures and test mechanics | Complete | SortedMap, cleanup appointments, and guarded replay are literate. | @@ -443,6 +444,7 @@ generic visitor behavior outside the named collectors. | Window-controller overlap | [shared infinite-query conformance](https://github.com/TanStack/db/blob/main/packages/db/tests/conformance/infinite-suite-oracle.ts), [React driver](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [Vue driver](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/infinite-query-conformance.test.ts), [Svelte driver](https://github.com/TanStack/db/blob/main/packages/svelte-db/tests/infinite-query-conformance.svelte.test.ts), [review record](oracle-reviews/dec-03-window-overlap.md) | Four or five ordered source rows, two window-settlement orders, and public snapshots after each settlement, a subscribed observer update, a detached controller read, and explicit recovery. Insertion and removal of the fifth row distinguish both continuation directions while an earlier error remains visible. The detached read follows a source change with no controller subscriber; its preloaded Collection retains the five-row window. The exact overlap uses the exported DB controller in each package realm. Framework hooks expose no controller preload, so this cell does not establish a hook scheduling path; direct hook paging remains in the adjacent shared scenarios. An unsubscribed controller has no notification claim. | | Infinite-query publication | [shared publication laws](https://github.com/TanStack/db/blob/main/packages/db/tests/conformance/infinite-suite-oracle.ts), [React render cuts](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-render-cuts-oracle.test.tsx), [Svelte hook tests](https://github.com/TanStack/db/blob/main/packages/svelte-db/tests/useLiveInfiniteQuery.svelte.test.ts), [review record](oracle-reviews/2026-10-05-infinite-first-paint.md) | Each driver records every value its framework published: React layout commits, Vue pre-flushes, and the Svelte construction value plus pre-effect runs. The shared suite checks that a synchronous source shows its ready first page on the first value, for a query callback and for a supplied collection that has not started, that every ready value in a fixed-source, fixed-query scenario equals the source prefix for its own page count, including an equal dependency replacement that keeps page depth, and that a mount requests an on-demand first window once, for a query callback and for a supplied collection whose finite window is wider than or equal to the one the hook needs, judged by the rows the on-demand source holds after mount. Under deferred acquisition, an on-demand first value is either the ready first page or a value with no rows, by framework: React and Svelte publish before the hook subscribes, Vue after. `preloaded-collection-first-paint` checks the documented route-loader pattern: a limited live query preloaded before mount shows its ready first page on the first value in every framework and asks the source for nothing more; a preload that does not count as demand never settles there. Pre-commit work has no shared form, because Vue setup is its commit. The React driver owns renders that never commit, StrictMode double renders (one first-window request on React 19), retries after GC, a mounted replacement whose startup error must reach the error boundary, and a collection kept from an abandoned render that enters terminal error before its query is selected again, each as under `useLiveQuery`. The React source ID reuse oracle drives the infinite hook too. Svelte owns construction and superseded recomputes before the subscribing effect. In both, the hook acquires an on-demand source as often as `useLiveQuery` does under the same history, for either input form, direct or behind a live-query Collection. On-demand requests from renders that never commit are now owned by the live-query deferred acquisition oracle; the React render-cut oracle checks that an abandoned render makes no acquisition. Open cells, owned here: React 18 StrictMode, where neither hook keeps refs across the double render; a React commit held after the 50 ms unsubscribed GC floor, where a controlled-premise controller witness shows no non-ready value but no React scheduling witness exists; ready values in scenarios that write the source or replace the query, which need a per-step source model; a join of eager and on-demand sources, which needs a zero-acquisition check before commit; a supplied collection with no `.limit`, which sends one unbounded request on `main` too, because the controller cannot narrow it before the source request; and Svelte `state_unsafe_mutation` when a start inside `$derived.by` synchronously writes rows a peer is waiting for, shared with Svelte's `useLiveQuery`, which a two-component probe reproduces and the deferred-acquisition follow-up removes. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-oracle.property.test.ts), [hash identity](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-identity-oracle.property.test.ts), [MultiSet consolidation](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/multiset-consolidate-oracle.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph-oracle.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph-oracle.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry-oracle.property.test.ts), [murmur stream](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/murmur-stream-oracle.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison-oracle.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils-oracle.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor-oracle.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update-oracle.property.test.ts), [BTree Map](https://github.com/TanStack/db/blob/main/packages/db/tests/btree-map-oracle.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts), [LIKE semantics](https://github.com/TanStack/db/blob/main/packages/db/tests/query/compiler/evaluators-oracle.test.ts) | Independent flat values, graph topology, algebraic laws, structural object keys in code-unit order (the flat-value, identity and consolidation grammars include keys that tie under locale collation), murmur byte framing against a byte-at-a-time reference (block mixing, the finalizer and symbol framing have no owner there), Map/group/sort recomputation, expression denotation, custom comparator dispatch/reference identity/index compatibility, LIKE wildcard refinement, compiled output bags, declared value-pair agreement between D2 equality keys and IR equality operands, and Collection collation with exact public scan/index order and one-index reuse across repeated reads. The auto-index cells cross BasicIndex and BTreeIndex; scan cells omit indexes. Both paths cover lexical and numeric `en-US` locale defaults plus an explicit ordinary locale override of a numeric default. An unindexed six-row work cell bounds Collection collation resolution to at most two reads per ordered scan: one index-path check and one scan clause. Other locales and ordered histories are outside this owner. The identity pair grammar covers primitives, signed zero, NaN/invalid Date, valid Date/timestamp, binary/Buffer, Temporal, nested references, symbols, and functions; fixed and sampled pairs do not prove every JavaScript value or hash collision freedom. Ref proxies are expressions rather than literal values in this lane. The deep-equality owner also pins that `deepEquals` ignores Map and Set insertion order, RegExp `lastIndex`, array holes, and typed-array class, the rules draft equality keeps. It also pins that objects of another class differ, that plain and null-prototype objects are one class, that a class instance without enumerable keys equals only itself, that URLs compare by `href`, and that typed-array elements compare like numbers. The hash-values owner checks equal-hash consequences for independently constructed flat values. Distinct type identities are checked by the hash-identity owner through `equalHashValues`, including forced digest collisions; distinct 32-bit digests are not promised. The hash-identity owner checks `hash` and `equalHashValues` against one spec-level identity model: primitives with signed zero, NaN, and bigint; reference leaves (functions, registered handles, `File`, binary over 128 bytes); Date, binary, Temporal, and RegExp headers; array length and holes; Map and Set insertion order; and object keys in any order, for acyclic values and for cycles through arrays, Map values, Sets, and plain objects. Map/Set order sensitivity is pinned current behavior, not a promise. Arrays from another realm, RegExps with a `NaN` `lastIndex`, getters, proxies, other cycle carriers, and pairs that differ only in a back-edge target are outside its grammar; pinned cases keep equality's cross-realm length check and strict RegExp field comparison. `hash-work.test.ts` pins how visited values and array/RegExp headers count toward the structural work cap, and that equality copies a shared Map's entries once per distinct pair ([code-weight hash dispatch review](oracle-reviews/code-weight-hash-dispatch.md)). Custom string indexes do not optimize ordinary ranges, and custom cursors remain unsupported. The LIKE owner covers boolean string matching and bounded work, not nullish three-valued logic or a general Unicode collation contract. Unsupported composite cursors reject. The MultiSet consolidation owner checks keyed, single-type, and structural identity, signed zero and NaN in keyed and single-number data, the retained first record, zero-sum removal, and input record contents. It does not assert output order. Its keyed grammar includes cross-type key and value collisions, non-finite numeric keys, the `\|` delimiter, symbol and function identity, and a direct-object/object-tuple delimiter control from #1948. Eight named collision witnesses and both generated campaigns are RED on `c879ba6d` and GREEN with the repair ([#1948 review record](oracle-reviews/issue-1948-keyed-consolidation.md)); the [code-weight consolidation review](oracle-reviews/code-weight-multiset-consolidation.md) records the earlier exclusion. The keyed reference uses direct value/reference equality, including SameValueZero for numeric keys. The unkeyed fallback keeps its original key and value domains. The direct `MultiSet` owner does not prove which `@tanstack/db` query shapes produce primitive keyed values or mixed-type source keys; that needs a live-query production witness. The index owner enforces one invalid-comparator law across both BasicIndex and BTreeIndex: for each comparator, accepted numeric prefix (including the empty prefix), and probe operation (add, update, eq/range lookup, take, build), the operation throws exactly when it receives a `NaN` or non-number result and never when every comparison is valid. A `signed infinity` comparator is the valid control, and a custom collation `compare` supplied through `compareOptions` is checked the same way. Two BTree Map split histories check inserted-key reads, exact whole-range payloads/order, size, and extrema, proving split placement follows the insertion index without a post-mutation comparison. A rejected index add or remove leaves the index refining its accepted rows; update and build are not atomic. At the Collection boundary, both index types and both write paths (optimistic insert, sync commit) crash the collection: status becomes `error` and the next mutation throws, so a usable collection never holds rows its subscribers were not told about. Open: a sync source can still commit writes on a collection in `error` state, which stores unpublished rows; this is existing `markError` behavior and needs a lifecycle-owner witness. Comparator transitivity is outside this evidence. | +| Mixed-kind ordered prefixes | [mixed-kind public-order oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/mixed-index-order-oracle.test.ts) | A separate unindexed ready Collection computes the public ordered two-row prefix for the same keyed rows as an eager BasicIndex or BTreeIndex Collection. Five histories cross number/bigint, boolean/number, object/string, and array/object values, with insert, update, and delete transitions into and out of mixed domains. Ascending and descending reads use one ascending physical index, so descending checks exercise ReverseIndex. After preload and after both source copies commit each step, exact direct and live-query IDs must match the scan. Same-kind checkpoints verify that safe indexes still serve ordered reads. The original type-name index order failed eight ascending public-prefix checks. A second numeric source Map models a finite on-demand provider: a bigint enters and leaves the visible prefix, deletion requires an uninstalled row, and an offset window reaches a cursor after the retained BTree index loses ordered-plan eligibility. Public IDs match the independent provider order at each settled window; controls withholding either requested row fail those assertions. This owner does not define a global precedence among unlike values, cover in-place value mutation or arbitrary mixed-kind pairs, or prove that every retained on-demand traversal is safe after its source domain changes. | | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts), [`in` evaluator](https://github.com/TanStack/db/blob/main/packages/db/tests/query/compiler/in-evaluator-oracle.property.test.ts) ([review](oracle-reviews/perf-in-constant-set.md)) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing was later removed in favor of pooled live queries; the routing mutants above are historical, and the same peer laws now check plain subscription dispatch. The unindexed snapshot prefilter was restored for compiled queries as one stored-row `eq` test on a string or boolean literal. Its property-visibility test also covers the reverse direction: a stored row with an inherited or non-enumerable field must still match `isUndefined` on that field, which a scan that evaluated the full predicate on stored rows fails. The `in` evaluator owner checks `in` against an independent linear reference over Dates, `NaN`, `-0`, BigInt, bytes, Temporal values, objects and nullish items, for constant and row-supplied lists. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Each step also checks that `get` returns what `getIterator` yields, in its order, since the two walk the storage separately. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Every history also runs with `prefixIdentity`, where values under one key with the same prefix are one value; mutants that merge unprefixed values or ignore the option in a prefix map fail. The [structural identity oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/structural-identity-oracle.property.test.ts) checks that `structuralIdentity` agrees with D2 value identity for equal representations (array subclasses, class instances, null prototypes, key order, non-enumerable symbols) and independent values, including separator-bearing strings and keys, enumerable symbol keys, Dates, RegExps, Maps, Sets, binary values, holes, deep nesting, and a cyclic value; an independent normal-form model decides equality and is checked against `equalHashValues`. Mutants that drop a string's or key's length, an array's length, or the Date and RegExp headers fail it; lowering the depth cap is equivalent, because the cap only bounds work. groupBy's use of it is owned by the incrementalization law: its string-extrema lane fails a string identity without a length prefix, and a focused groupBy witness fails when the group key is left out of the identity. Compaction, presence tracking, and structural payloads are outside this owner; a focused test checks that an index built without presence (as reduce builds its indexes) throws when presence is read; the [reduce fast-path oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/reduce-fast-path-oracle.property.test.ts) compares the fast path's emitted entries one by one with the general path's, over keys that move between the paths, hash-merged outputs, -0, and zero and negative multiplicities, while an unchanged `NaN` output is owned by a focused reduce test; the join operator tests and incrementalization law own operator behavior. | @@ -459,6 +461,108 @@ generic visitor behavior outside the named collectors. | Boundary refinements | [cleanup/restart](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-cleanup-restart-oracle.test.ts), [idle cleanup](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-idle-cleanup.test.ts), [issue #1891 async-cleanup review](oracle-reviews/issue-1891-async-cleanup.md), [issue #1891 cleanup-start follow-up](oracle-reviews/issue-1891-cleanup-start-followup.md), [metadata publication](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-metadata-publication-oracle.property.test.ts), [row metadata composition](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-row-metadata-composition-oracle.test.ts), [nested-begin metadata rebuild review](oracle-reviews/2026-10-06-nested-begin-metadata-rebuild.md), [state mutation round 4](oracle-reviews/2026-10-06-state-mutation-round-4.md), [state retention](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-state-retention-oracle.property.test.ts), [acquisition cells](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-subscription-lifecycle-oracle.test.ts), [D2 source reconciliation](https://github.com/TanStack/db/blob/main/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts), [top-K support windows](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/topk-support-window-oracle.test.ts), [nested Query work](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/includes-work-counter-oracle.test.ts) | Explicit lifecycle products, independent source maps and weighted relations, exact publication cuts, support/multiplicity, and value-plus-work observations. These refine the larger subsystem models; they do not replace them. | | Small structures and test mechanics | [SortedMap](https://github.com/TanStack/db/blob/main/packages/db/tests/SortedMap-oracle.test.ts), [cleanup queue](https://github.com/TanStack/db/blob/main/packages/db/tests/cleanup-queue-oracle.property.test.ts), [live-query GC clock](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-gc-clock.test.ts), [guarded replay](https://github.com/TanStack/db/blob/main/packages/db/tests/oracle-replay.test.ts) | Map/full-sort and elapsed appointment-list models with executed target/seed/path checks. SortedMap also checks deferred bulk writes, array-movement work, scan reuse, live iterators, runtime `undefined` keys, and public single-transaction and queued-transaction Collection sync paths. It does not settle the contract for comparator-observed values mutated after publication or every legal iterator/mutation interleaving. Wall-clock steps do not move due times, and a focused public witness checks source subscription release. Callback-reentrant scheduling, OS suspend/resume, environments without `performance.now()`, and fake/real Performance clock replacement while GC is pending remain outside this coverage. Tests must drain or reset pending GC before switching timer providers. | +## Unit-test retirement transfers in PR #2116 + +The [direct index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update-oracle.property.test.ts) +owns the seven behaviors moved from index unit tests: an absent lower bound +with an exclusive flag; exact membership through comparator-equal updates; +`null` and `undefined` exact and `IN` membership, including repeated owners; +undefined endpoint and cursor traversal; and explicit, omitted, and +double-undefined range bounds. A source Map predicts exact key sets and +ordered reads after bounded direct actions on BasicIndex and BTreeIndex. A +full-build witness catches native `Array.sort` moving raw `undefined` last. +The [round 1](oracle-reviews/index-unit-retirement-round-1.md), +[round 2](oracle-reviews/index-unit-retirement-round-2.md), +[round 3](oracle-reviews/index-unit-retirement-round-3.md), and +[round 4](oracle-reviews/index-unit-retirement-round-4.md) ledgers retain each +retired assertion. The [build review](oracle-reviews/basic-index-nullish-build.md) +records the direct RED/GREEN repair. A separate mixed number/string lane +checks that full direct traversals and one indexed, bounded public Collection +snapshot return each live source key once after deletion and after a direct +delete/update/re-add history. Two more direct histories keep an ordinary string +beside ordered ZonedDateTime or Duration keys and verify key conservation after +one Temporal deletion. An en-US locale and a custom string-collation cell compare +the first bounded public row through a scan and each index when an ordinary +string begins the internal encoding prefix. An `origin/main` probe failed the +default-locale public prefix case. The pre-classifier PR comparator failed all +four public prefix cases and both BTree Temporal-deletion histories. This owner +also compares complete forward and reverse reads through `IndexInterface`, +without prescribing index buckets or a BTree representative. A fixed churn +witness holds one live row while 16, 64, or 256 earlier distinct values are +added and removed; its public `takeFromStart(1)` result and comparator-call +growth reject retained empty buckets that add comparator work. This is a bounded +read-work check, not a memory bound or a general complexity proof. A deliberate +empty-bucket mutant failed at the intended comparator-count comparison. +The invalid-comparator owner checks rejection exactly when a comparison +returns a non-number or `NaN` and checks rejected removal atomicity; it does +not require a comparison merely because one comparator-equal row is removed. +The binary lane checks fresh equal-content removal, re-addition, and +replacement beside another byte value. It compares complete forward and +reverse key reads to a source Map. Separate short and 8193-byte pairs check +exact membership and complete forward and reverse key conservation when only +the last byte differs. An intermediate PR BTree rewrite lost a surviving key +after one equal-content owner left; the expanded lane rejected that result. +A separate two-value Date, Temporal.Instant, and Uint8Array callback lane +checks legacy scalar input kinds on direct add only. Arbitrary callbacks and +update, build, and range paths are outside that lane. +This owner does not prescribe mixed-kind order or cover binary range ordering, +all JavaScript value kinds, or general Collection publication histories. + +The [comparison oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison-oracle.property.test.ts) +owns the transferred same-realm NaN and invalid-Date default-order cases, +plus ordinary PlainDate calendar and ZonedDateTime instant order. Its +independent supplied values predict direct comparator signs and ties. The +[round 5](oracle-reviews/comparison-unit-retirement-round-5.md) and +[round 6](oracle-reviews/comparison-unit-retirement-round-6.md) ledgers name +the retired examples. Mixed-kind order, cross-realm values, modified built-ins, +and public indexed reads are outside this transfer. + +The [deep-equality oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/utils-oracle.property.test.ts) +owns the transferred same-realm Date and RegExp equality boundaries: equal +copies, changed values, and inequality with timestamp, ISO, or source text. +The [round 7 ledger](oracle-reviews/utils-unit-retirement-round-7.md) retains +the retired examples. Caller-modified built-ins and Collection publication +are outside this direct-return owner. + +The [Duration query equality oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/duration-query-equality-oracle.test.ts) +uses ten public fields to predict direct BasicIndex/BTreeIndex exact lookups and +public `eq` snapshots before and after indexing. The `in` evaluator owner also +checks same-display Duration forms through constant and row-supplied lists. +The deep-equality and Collection change-event history owners check a Duration +field replacement and its subscriber mirror. These fixed histories do not +establish arbitrary multirow Duration updates or live-query publication. + +The [Temporal index order oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/temporal-index-order-oracle.test.ts) +owns ZonedDateTime instant ordering and time-only Duration ordering for direct +BasicIndex/BTreeIndex reads and bounded public first-row reads. It separates +exact Duration fields from order ties, including exclusive cursor behavior. +An extended-year PlainDate lane checks calendar order under lexical string +collation through ready scan and direct indexed snapshots, with eager live-query +preload as parity evidence. The old indexed direct path chose the wrong year. +An exception lane requires ready scan, BasicIndex, and BTreeIndex direct reads +and eager live-query preloads to reject mixed PlainDate/PlainDateTime values and +distinct PlainMonthDays, as changelog #1519 specifies. The old indexed direct +path returned a row instead. Calendar-unit Duration order, other extended-year +histories, on-demand exception publication, arbitrary updates, and custom index +callbacks remain outside this owner. + +The [custom-index selection oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/custom-index-order-selection-oracle.test.ts) +checks that a reverse manual index cannot change an ascending limited public +snapshot or live-query result. Eager and prepopulated on-demand sources with +three distinct numeric ranks and a two-row window compare public IDs after +preload. The reviewer withdrew a claimed wrong live-query result after fixing +its scratch test's cross-checkout `@tanstack/db-ivm` resolution. The witness +makes no claim about the chosen physical plan or per-row custom callback order +inside a comparator tie. Provider paging from an incomplete on-demand source +and later updates remain outside this witness. + +The [PR #2116 confirmed-fixes review](oracle-reviews/pr-2116-confirmed-fixes.md) +records baseline failures, hostile controls, and the bounded scope of these +repairs. The [deferred findings record](oracle-reviews/pr-2116-deferred-findings.md) +tracks remaining design and coverage questions, including valid Temporal kinds +without public order and mixed-kind query order. Earlier PR-branch +counterexamples do not by themselves establish an `origin/main` failure. + The Query DB ownership oracle owns the direct-write no-automatic-refetch law ([review](oracle-reviews/issue-1935-direct-write-refetch.md)). Its real QueryClient witness loads ten disjoint on-demand subsets, then checks diff --git a/docs/contributing/oracle-reviews/basic-index-nullish-build.md b/docs/contributing/oracle-reviews/basic-index-nullish-build.md new file mode 100644 index 0000000000..5ab6dc279e --- /dev/null +++ b/docs/contributing/oracle-reviews/basic-index-nullish-build.md @@ -0,0 +1,152 @@ +# BasicIndex build with a nullish value + +At PR head `175adce0cb32e29e5838853c821ee5c56fe0089a`, a focused runtime +witness built a direct BasicIndex from `undefined` and `1`. The default +comparator places `undefined` first. `BasicIndex.build` returned +`takeFromStart(2) = [one, undefined]` and an empty inclusive upper range +bounded by `undefined`. Incremental `BasicIndex.add` and both BTreeIndex paths +returned `[undefined, one]` and the expected range. A recording comparator +received no calls during the BasicIndex build. The build path is also reached +when `Collection.createIndex` builds an index over existing rows. + +The accepted law is the default ascending, nulls-first order documented by +`DEFAULT_COMPARE_OPTIONS`, `defaultComparator`, and the direct `IndexInterface` +contract. The primary `index-update-oracle.property.test.ts` already models +nullish positions with a source Map, but its nullish history grammar previously +used add/update/remove only; its generated build grammar used numeric values. +The smallest missing cell is `undefined` plus `1` through `BasicIndex.build`, +observed with `takeFromStart(1)` and an explicit upper range. A two-order +fixed build history and generated nullish builds now use the same Map checker +after each returned action. The Map distinguishes exact null and undefined +membership while its ordered projection treats them as one comparator +position. Raw duplicate source keys are resolved before build; mixed value +classes and custom comparators remain outside this claim. + +## RED evidence before the production change + +The new fixed oracle witness failed on the unchanged implementation: +`BasicIndex nullish direct index refinement > build preserves nulls-first +ordered reads and ranges` compared `takeFromStart(1)`, receiving `[one]` +against source-model `[undefined]` at the intended observation checkpoint. +The BTreeIndex version passed. The command's focused run had one failed test, +one passed test, and no type errors. This is a product mismatch, not a setup +failure or a changed expectation. + +Native `Array.sort` moved a raw `undefined` element to the end without invoking +the supplied comparator. `BasicIndex.build` now sorts wrapper records, so every +stored value reaches the configured comparator, then unwraps the sorted values. +The fixed two-order build history and generated put/delete/build histories +pass through the same independent Map checker after each returned action. + +## GREEN evidence and hostile review + +The complete direct index oracle passed 412 tests with no type errors after the +repair. The fixed history includes both source insertion orders and a later +build with `undefined`, `null`, and a defined value. The generated grammar can +replace an existing source Map through build. The fresh hostile reviewer +`/root/review_nullish_build_oracle` inspected the original RED witness, the +model, both index implementations, and the repair. It found no blocker for the +bounded direct-index law. It did not claim that a Collection query using a +newly built index has been observed publicly. + +The oracle's model derives exact membership and sorted source keys from the +source Map, not from either index. A build that leaves `undefined` last fails +`takeFromStart(1)` at the returned-action checkpoint. Range and cursor probes +also compare against the Map after each action. The direct-index law excludes +raw duplicate-entry builds, mixed string/number order, custom comparators, +Collection queries, and work complexity. Its fixed and generated build cells +do not establish every possible source history. The exact committed-head +review is recorded in a later entry. + +## Exact committed-head hostile recheck + +The reviewed commit is `9a145fba130f27c914a4a1fd4f44c5871a917483`. +`/root/exact_review_index_build` read the source-isolated index retirement +ledgers, the index oracle and coverage map, the direct index implementations, +and the default comparator and interface contracts. The commit preserves the +earlier RED witness: its parent still sorts raw values, so native `Array.sort` +moves `undefined` behind `1` without calling the comparator. The repaired +build sorts objects that hold the values; this invokes the checked comparator +for that pair and leaves equal comparator positions stable. This recheck +inspected the parent implementation and the recorded RED run; it did not +execute the parent commit again. + +The committed direct index oracle passed 412/412 tests with no type errors. +The guarded replay of `index-update.nullish-membership` at seed `20261012`, +path `0`, reached both index classes for 100 runs each and passed. `git show +--check` passed, and the worktree was clean after those runs. No new unit test +was retired by this build repair. The earlier retirement ledgers still delimit +the assertions removed from their source files. + +| Requirement | Exact-head result | +| --- | --- | +| ORC-001 | `DEFAULT_COMPARE_OPTIONS`, `defaultComparator`, and the direct `IndexInterface` methods support default ascending, nulls-first returned results. The interface lists methods but does not itself spell out this order. The claim is limited to direct default-option reads over `undefined`, `null`, and positive numbers. | +| ORC-002 | A source Map owns exact key/value pairs. Its `nullishPosition` projection freshly sorts and filters rows without production normalization, comparator, index storage, or traversal. | +| ORC-003 | Opening and adjacent oracle prose states the law and limits, Map rule, legal put/delete/build histories, direct BasicIndex/BTreeIndex path, returned arrays and sets, and the after-action comparison cut. | +| ORC-004 | The fixed trace reconstructs `undefined` plus `1` in both build insertion orders, a put after build, then a four-key replacement with `undefined`, `null`, and `1`. Generated histories use three keys, five values, 1–40 actions, and builds of at most three entries. Removing build loses the full-sort path; removing `undefined` loses the native-sort distinction; removing defined values loses the nullish boundary; removing replacement loses stale-owner detection. Duplicate generated keys are resolved by the Map before build. Raw duplicate-entry builds, other value domains, and direct updates of absent keys are excluded. | +| ORC-005 | The oracle invokes real direct `build`, `add`, `update`, and `remove` methods, then compares exact membership sets, complete forward range sets, and ordered key arrays after each returned action. `takeFromStart(1)` observes the original defect. No public Collection query is claimed. | +| ORC-006 | The recorded unchanged-production run failed at the `takeFromStart(1)` refinement comparison, with `[one]` instead of `[undefined]`; this is an assertion failure at the intended cut. The repaired version passes. No additional production mutant was run in this recheck. | +| ORC-007 | Fixed-seed and seedless lanes share the generated action grammar, recorder, comparison, and 100-run budget. The exact-head guarded seed/path replay reached two 100-run executions. The earlier round-4 ledger records a captured failure replay for this named property, before the build action changed its arbitrary. A captured failing seed/path for the revised arbitrary was not independently checked in this recheck; that replay-fidelity cell remains open. | +| ORC-008 | The existing Map retains key/value ownership because later put and delete actions distinguish owners. Build adds no independent model state; it replaces that Map. | +| ORC-009 | Model `put` maps to add or update according to Map ownership, `delete` to remove when owned, and `build` to a full replacement. The model-only comparator position combines null and undefined for order while exact membership keeps them separate. | +| ORC-010 | The action loop has no external resources. The guarded replay retains its named witness and removes its private temporary channel; the recorded original failure remains distinct from a setup or cleanup failure. | +| ORC-011 | No plausible fault shared by this bounded source-Map rule and production was identified. Nullish position follows the accepted default comparator contract; mixed domains and custom comparison need separate laws. | +| ORC-012 | This append-only entry ties the exact-head review, executed GREEN result, inspected RED path, guide audit, and limits to the reviewed commit. The revised-arbitrary failure replay named under ORC-007 is the remaining conformance evidence gap. | +| ORC-013 | The two-entry `undefined`/`1` build reaches the law's premise. The first fixed observation rejects the plausible wrong rule “native sort may place undefined last”; reversing source insertion order distinguishes an accidental insertion-order pass. | +| ORC-014 | No controlled provider or host premise supports a claim about real-provider behavior. Direct index results are the stated boundary. | + +There is no blocker for the bounded direct-index code or its source-Map +comparison. This result does not establish Collection query selection after +`createIndex`, custom comparator behavior, mixed string/number ordering, raw +duplicate-entry builds, all range shapes, or work complexity. The revised +arbitrary's captured-failure replay remains to be recorded before claiming +complete ORC-007 conformance for the expanded generated property. + +## Revised-grammar failure replay calibration + +This follow-up closes the ORC-007 evidence gap above for the oracle at reviewed +commit `9a145fba130f27c914a4a1fd4f44c5871a917483`. A temporary wrong +expected result was inserted after the nullish action checker: whenever a +generated `build` returned, it required `takeFromStart(1)` to equal the +impossible key `calibration-only-missing-key`. This deliberately faulty +expectation did not modify production or the action arbitrary. + +The seedless generated lane failed at the returned-build endpoint comparison +for BasicIndex after one test, seed `1770427226`, shrink path `0:0`. Its +counterexample was one empty build; the actual endpoint was `[]`, while the +temporary expectation was `[calibration-only-missing-key]`. BTreeIndex also +failed at the same temporary assertion in its seedless lane. Guarded direct +replay of `index-update.nullish-membership` with the BasicIndex test selector, +seed `1770427226`, and path `0:0` reached exactly one property execution. Its +witness recorded one run and `failed: true`; Vitest and the replay guard exited +nonzero with the same endpoint assertion. This is an assertion failure at the +intended comparison, not a setup error or timeout. It demonstrates replay +fidelity for the revised arbitrary; the empty-build case is a replay control, +not the nonempty nullish-order product witness. + +After removing the temporary expectation, the same guarded seed/path replay +reached one BasicIndex execution, recorded `failed: false`, and passed with no +type errors. `git diff --exit-code -- +packages/db/tests/index-update-oracle.property.test.ts` confirmed that the +oracle file again matched the reviewed commit. No calibration fault remains. +The preceding ORC-007 and ORC-012 gaps are closed by this executed evidence. + +## Transfer-only narrowing + +The later production refactor groups each normalized build value during the +existing entry pass. It removes the intermediate entries array and keeps the +wrapper sort that fixes the nullish position. A fresh hostile recheck read +the source Map model, original unit ledgers, and current build code. Its +focused index, work, and Collection-index run passed 535 tests with no type +errors. The recheck found no lost direct-index assertion after the grouping +change. It did not rerun the original wrong-sort mutant or extend the law to +raw duplicate source keys or custom callbacks within one normalized value. + +At exact code commit `24a61a8d5b6cef06d488c490e39f351b4d55fb68`, a +fresh hostile reviewer rechecked the one-pass build against the seven retired +index law families and the nullish fixed history. The source Map, wrapper +sort, and direct observations remained intact after the commit hook. A +focused index, work, and Collection-index run passed 535 tests with no type +errors. The reviewer found no in-scope loss. This recheck did not rerun the +original wrong-sort mutant; the earlier RED run remains the sensitivity +evidence. diff --git a/docs/contributing/oracle-reviews/comparison-unit-retirement-round-5.md b/docs/contributing/oracle-reviews/comparison-unit-retirement-round-5.md new file mode 100644 index 0000000000..337c0c7738 --- /dev/null +++ b/docs/contributing/oracle-reviews/comparison-unit-retirement-round-5.md @@ -0,0 +1,72 @@ +# Comparison unit retirement: non-finite default order + +A source-isolated Field Lab loss audit compared `comparison.test.ts` with +`comparison-oracle.property.test.ts` at frozen head +`f6857fee9de59dc82cbe596ca7df4a18152004ed`. The receiver initially +excluded NaN and invalid Dates and did not observe `defaultComparator` sorting. +The audit froze the five blocks below before retirement. The remaining three +unit tests cover symbol order and `compareValues` behavior at another entry +point. + +| Retired assertion | Receiving check | +| --- | --- | +| `ascComparator(NaN, 5)` is positive and the reverse is negative. | Tagged finite and non-finite number pairs check both signs. | +| `ascComparator(NaN, NaN)` returns exact zero. | Duplicate non-finite pairs check `toBe(0)`, including the sign of zero. | +| Sorting `[3, NaN, 1, 5, NaN]` puts the finite values in order and both NaNs last. | A fixed tagged history predicts every stable sorted row ID. | +| Sorting `[5, NaN, null, 1]` puts null first and NaN last. | Fixed and generated histories combine optional null with non-finite values. | +| An invalid Date sorts after a valid 2023-01-01 Date in both directions. | A fixed history preserves that day; generated valid epoch days and invalid Dates check both signs and sorted IDs. | + +Changelog #1582 promises PostgreSQL-style NaN and invalid-Date ordering. +`DEFAULT_COMPARE_OPTIONS` and `defaultComparator` supply the default ascending, +nulls-first path. The independent model receives tags for null, finite, and +non-finite values; it compares tag rank and supplied finite values. It never +uses production's unorderable classifier to predict a result. The driver calls +`ascComparator` and sorts with `defaultComparator`. It checks direct signs and +exact zeros at return, then compares every ordered row ID after sorting. + +The generated grammar uses 1–5 distinct finite numbers or valid Dates in +epoch-day range -20,000..20,000, one or two NaNs or invalid Dates, and optional +null. Number and Date histories remain separate. Mixed number/Date order, +Temporal values, symbols, other options, `compareValues`, and public query or +index paths are outside this transfer. + +## Review and replay + +The direct wrong-result control ties a non-finite value to a finite value and +fails the sign check. A separate wrong-sort control ties NaN during sorting +and fails the ordered-ID check. Fixed-seed and seedless campaigns share the +same 100-case property. Direct property 23 replay at seed `20260911`, path +`0`, passed. + +The latest hostile reviewer read the retired unit blocks, ledger, model, +production comparator, and coverage map. It found no lost assertion at the +declared direct and sort checkpoints. The focused comparison suite passed +100 tests with no type errors. Earlier PR revisions explored cross-realm and +long-byte behavior; those experiments are outside this transfer and are not +claimed by this record. + +## Oracle guide boundary + +The opening law and ledger state authority and limits (ORC-001). Tagged +finite values and source ranks give an independent computation (ORC-002); +the oracle places law, model, grammar, driver, observation, and checkpoint +beside their code (ORC-003). Fixed operands reconstruct the five old blocks, +and generated finite, duplicate non-finite, and null cases exercise nearby +histories (ORC-004). Direct comparator returns and sorted row IDs are the +declared observations (ORC-005). The two wrong-result controls +fail at those comparisons (ORC-006 and ORC-013). The replay above checks the +shared fixed-seed and seedless harness (ORC-007 and ORC-010). Model tags are +input labels, with no state transition or host provider (ORC-008, ORC-009, +and ORC-014). This ledger records the loss audit, hostile challenge, and +bounded result (ORC-011 and ORC-012); it makes no claim about other value +domains. + +## Exact code-commit review + +At `24a61a8d5b6cef06d488c490e39f351b4d55fb68`, a fresh hostile reviewer +compared all five retired blocks with the committed tagged model, direct +comparator checks, and sorted row IDs. It found no lost assertion. The +committed production and oracle blobs match the reviewed working tree; the +commit hook did not change them. A focused comparison and evaluator run +passed 236 tests with no type errors. This verdict does not extend to +`compareValues` or public query and index paths. diff --git a/docs/contributing/oracle-reviews/comparison-unit-retirement-round-6.md b/docs/contributing/oracle-reviews/comparison-unit-retirement-round-6.md new file mode 100644 index 0000000000..c503924e11 --- /dev/null +++ b/docs/contributing/oracle-reviews/comparison-unit-retirement-round-6.md @@ -0,0 +1,72 @@ +# Comparison unit retirement: Temporal order + +A source-isolated Field Lab loss audit compared two Temporal blocks in +`packages/db/tests/comparison.test.ts` with the comparison oracle at frozen +head `c038b2b673a601d3bfbc70261f2cbb61f6f934ea`. The receiver had no +Temporal input. These four observations were frozen before the blocks were +removed. The symbol block and two `compareValues` NaN blocks stay in the unit +file because they have different obligations. + +| Retired block | Original observation | Receiving oracle | +| --- | --- | --- | +| PlainDate calendar order | January 2024 precedes June 2024; reversing the operands is positive; a fresh January value compares at exact zero. | Supplied ISO year, month, and day fields predict both signs and exact equal-date zero. Fixed operands reconstruct January and June; generated fields vary each part. | +| ZonedDateTime instant order | 06:30 UTC and 12:00 +05:30 representations of one instant compare at exact zero. | Supplied epoch-nanosecond offsets predict both signs and zeros across four offset zones, including one-nanosecond neighbors. | + +Changelog #1519 promises Temporal static comparison for `orderBy` and range +operators. It specifies instant order for ZonedDateTime, while `eq` retains +zone as part of identity. The receiver models PlainDate calendar fields and +ZonedDateTime instants independently of production dispatch and Temporal's +static comparator. Its driver builds ordinary values from one polyfill copy, +calls `ascComparator` with default options, and compares each direct return. +It does not claim `compareValues`, index, public query, or provider behavior. + +The grammar uses ISO PlainDates in years 2000–2030 with days 1–28. It uses +ZonedDateTimes within two seconds of a fixed 2024 instant in UTC, +05:30, +-05:00, and +14:00. Equal-value branches create separate objects; adjacent +instants differ by one nanosecond. Mixed Temporal kinds, unsupported +PlainMonthDay order, subclasses, altered constructors or prototypes, and +mixed native/polyfill implementations are outside this transfer. + +## Review and replay + +The fixed cases reconstruct all four old assertions. A wrong comparator +that uses zoned display text fails the equal-instant sign check; a +millisecond-truncating comparator fails the adjacent-instant check. ISO +PlainDate display text has the same order as the tested calendar fields, so +the text-order control only distinguishes the zoned law. The generated +properties run fixed-seed and seedless 100-case campaigns. Direct replays +38 and 39 at seed `20260911`, path `0`, passed. + +The latest hostile reviewer read the retired assertions, model, production +comparator, and coverage map. It found no lost assertion within the direct +`ascComparator` scope. The focused comparison suite passed 100 tests with +no type errors. The production change removes a tag-wide dispatch cache. +For ordinary Temporal objects this preserves the required static comparison +while reducing code. Earlier PR revisions tested caller-modified constructors, +spoofed tags, and subclasses; the maintainer declined guarantees for modified +standard values, and those witnesses are not part of this transfer. + +## Oracle guide boundary + +The opening contract identifies #1519 and the direct-return limit (ORC-001). +Supplied fields and offsets determine the expected result without importing +production classification or comparison (ORC-002). The oracle places model, +grammar, driver, observation, and comparison beside explanatory prose +(ORC-003). Fixed operands and generated year, month, day, zone, equal-instant, +and adjacent-nanosecond dimensions cover the bounded grammar (ORC-004). +`ascComparator` returns are checked immediately (ORC-005). Wrong text and +truncated-instant results fail those checks (ORC-006 and ORC-013). Direct +seed/path replay verifies the shared campaign harness (ORC-007 and ORC-010). +The tuple and instant model has no mutable state or host provider (ORC-008, +ORC-009, and ORC-014). This record preserves the loss audit, hostile review, +and explicit limits (ORC-011 and ORC-012); it makes no cross-boundary claim. + +## Exact code-commit review + +At `24a61a8d5b6cef06d488c490e39f351b4d55fb68`, a fresh hostile reviewer +reconstructed the three PlainDate assertions and equal-instant zoned +assertion from the committed oracle. The comparator and oracle blobs match +the versions reviewed before commit. Its focused comparison and evaluator +run passed 236 tests with no type errors. It found no lost ordinary +same-realm assertion. Modified Temporal inputs, `compareValues`, and public +query or index paths remain outside this verdict. diff --git a/docs/contributing/oracle-reviews/index-unit-retirement-round-1.md b/docs/contributing/oracle-reviews/index-unit-retirement-round-1.md new file mode 100644 index 0000000000..f570ee4ed7 --- /dev/null +++ b/docs/contributing/oracle-reviews/index-unit-retirement-round-1.md @@ -0,0 +1,94 @@ +# Index unit-test retirement: two-block ledger + +Base: `f6857fee9de59dc82cbe596ca7df4a18152004ed` (`origin/main` at worktree creation). +This ledger names the exact legacy assertions proposed for retirement. The +receiving owner is `packages/db/tests/index-update-oracle.property.test.ts`. +The other blocks in each legacy file remain outside this change. + +## Upper-only range ignores an absent lower bound + +- **Legacy source:** `packages/db/tests/btree-index-undefined-values.test.ts`, + `should not drop the minimum key when an upper-only range is exclusive on + the (absent) lower bound` (original line 252). It inserts values 1, 5, 10, + calls `rangeQuery({ to: 10, fromInclusive: false })`, and requires all three + keys. Its observation is the returned direct BTreeIndex set. +- **Law and authority:** An absent bound denotes the start of the index + (`packages/db/src/query/live/ARCHITECTURE.md`, Identity); the documented + `IndexInterface.rangeQuery` accepts separate bounds and inclusivity flags. + An inclusivity flag cannot exclude a value at a bound the caller did not + supply. This law says nothing about internal tree traversal or runtime cost. +- **Independent model and history:** The receiving oracle's source `Map` keeps + key/value ownership. For every upper bound -2, 0, 2, it filters the Map with + `value <= upper`; it makes no lower comparison for an absent lower bound. + Paired reads with an explicit exclusive `from` require `value > lower`, + preserving the distinction stated in the old test's comment. + The existing fixed and random campaigns apply add, same-key update, delete, + and full build to both BasicIndex and BTreeIndex, then check the upper-only + and explicit-bound reads after each action, including empty and + changed-minimum states. +- **Distinction and checkpoint:** A design that applies + `fromInclusive: false` to the implicit minimum drops a modeled key. The + comparison checks the exact returned set after each direct index action. + One wrong-output control removes the minimum from the upper-only read; + another adds the excluded boundary to an explicit-bound read. Both must + fail their intended comparison. +- **Retirement gate:** The new owner and both wrong-output controls passed. + `/root/review_absent_bound` attacked the original and revised claims, + executed the model and controls, and found no blocking survivor in the + bounded numeric scope. The named legacy block was removed; its sibling + blocks remain. + +## Comparator-equal update moves exact membership + +- **Legacy source:** `packages/db/tests/index-update-short-circuit.test.ts`, + `does not use comparator equality to skip an update` (original line 91). + With a case-insensitive comparator, it changes one key from `A` to `a` and + checks that exact `A` lookup becomes empty and exact `a` lookup owns the key. +- **Law and authority:** An ordered index keeps separate exact equality + buckets for values at one comparator position + (`packages/db/src/query/live/ARCHITECTURE.md`, Identity; the existing + comparator-group index oracle). Comparator equality alone does not authorize + skipping a source value replacement. This does not promise bucket object + identity or prohibit harmless maintenance work. +- **Independent model and history:** A source `Map` retains exact strings; + fixed input facts put `A/a` in one order group and `B/b` in another. The + receiving oracle checks both `equalityLookup` and `lookup('eq')`, plus + comparator-group range sets, after each direct action on BasicIndex and + BTreeIndex. The fixed trace updates a singleton `A → a → A` before adding + peers, then checks populated same-group and cross-group updates. Its + existing fixed/random + custom-comparator property now also replaces an object with a fresh object + in the same group before crossing to another group. The group IDs and exact + object handles are input facts, not production classifications. +- **Distinction and checkpoint:** A wrong design that skips a comparator-equal + `update` leaves the old exact lookup populated at the returned-action cut. + A singleton no-op update control must fail the same source-Map comparison. + A second control makes `lookup('eq')` return the comparator group while leaving + `equalityLookup` intact; it must fail as well. The fixed string trace + preserves the original value-domain and lookup-dispatch path; the generated + object lane probes the general distinction after representative retirement. +- **Retirement gate:** The string and generated witnesses and both controls + passed. `/root/review_equal_update` found two gaps in the first draft: it + checked only `equalityLookup`, and it updated only a populated index. The + revised witness exercises `lookup('eq')` and the original singleton path; + the reviewer re-ran the relevant cases and found no remaining blocker. + The named legacy block was removed; its sibling blocks remain. + +## Execution and workflow result + +The post-retirement run of the receiving oracle and both remaining legacy +files passed: 397 tests, zero type errors. Before retirement, the same three +files passed with both old blocks present. The new numeric and string controls +deliberately inject wrong returned results or skipped work and each reaches an +assertion failure at the intended public observation. No production defect was +demonstrated in these two blocks, so production code was not changed. + +This round exposed two useful review gates for the broader cleanup. A +same-group update on populated data did not cover the original singleton +history, and checking `equalityLookup` did not cover `lookup('eq')` dispatch. +Both gaps survived the author's first green run and were found by an +independent reviewer. The generated object fixture also initially reused a +retired key; distinct reuse keys repaired that model-fixture error before the +final run. Future blocks should record the exact public method, smallest +state, old comment's law, and a wrong-design control before deletion. A +passing test count by itself is not a retirement gate. diff --git a/docs/contributing/oracle-reviews/index-unit-retirement-round-2.md b/docs/contributing/oracle-reviews/index-unit-retirement-round-2.md new file mode 100644 index 0000000000..e14bad4bfd --- /dev/null +++ b/docs/contributing/oracle-reviews/index-unit-retirement-round-2.md @@ -0,0 +1,92 @@ +# Index unit retirement: nullish exact membership + +The source-isolated loss audit used baseline `73a5d4ec4`. The receiving owner is +`packages/db/tests/index-update-oracle.property.test.ts`. No source block below +was an exact witness in that baseline: its generated update values were numeric, +and its fixed nullish case read ranges and cursors rather than equality buckets. + +## Transfer ledger before retirement + +| Legacy block | Smallest source state and direct action | Exact source observation and law | Plausible wrong result | +| --- | --- | --- | --- | +| `index-update-short-circuit.test.ts`, `does not conflate undefined and null` | BasicIndex and BTreeIndex: fresh `a=undefined`; update `a→null` | At update return, `lookup('eq', undefined)=∅` and `lookup('eq', null)={a}`. The title states distinct exact membership. | A loose-nullish shortcut leaves `a` in the old bucket; a dispatch that returns the comparator group includes it in both. | +| `btree-index-undefined-values.test.ts`, `should update item from defined to undefined value` | BTreeIndex: fresh `a=1`; update `a→undefined` | At update return, `equalityLookup(1)=∅` and `equalityLookup(undefined)={a}`. | The old key retires but the new undefined key is lost. | +| Same file, `should update item from undefined to defined value` | BTreeIndex: fresh `a=undefined`; update `a→1` | At update return, `equalityLookup(undefined)=∅` and `equalityLookup(1)={a}`. | The new key arrives but the undefined membership remains. | +| Same file, `should update item from undefined to undefined (same value)` | BTreeIndex: fresh `a=undefined`; update `a→undefined` | At update return, `equalityLookup(undefined)={a}`. This checks conservation, not bucket identity. | Remove without re-add drops `a`. | +| Same file, `should handle mixed undefined and null values` | BTreeIndex: fresh adds `null1=null`, `undef1=undefined`, `num1=1` | The two exact lookups return their respective singleton sets. The block checks no ordering despite its enclosing heading. | Returning the shared nullish comparator group for either value includes the other key. | + +The BTree file's opening infinite-loop and sentinel comments concern traversal; +they do not establish an equality law. `IndexInterface` exposes both lookup +methods, and the established direct-index tests distinguish nullish exact +values. The receiving oracle already states the separation between exact +membership and comparator position. This transfer is limited to direct index +methods. A live-query WHERE comparison with a nullish operand follows +three-valued logic, so it is outside this law. + +## Receiving law and evidence to check + +A plain source `Map` retains each key's exact value. The oracle recomputes +equality buckets without index normalization or comparator classification. +It drives each source action through BasicIndex and BTreeIndex and checks +`keyCount`, indexed keys, `equalityLookup`, and `lookup('eq')` after return. +Five fresh singleton or coexisting-add histories reconstruct the source cases; +an additional `null→undefined` singleton and fixed/random put/delete histories +challenge nearby legal transitions. The generated domain is three keys and +`undefined`, `null`, `1`, `2`. A direct update of an absent source key is +excluded; the driver uses `add` for that source action. + +A skipped `undefined→null` update and a wrong `lookup('eq')` union are intended +to fail the model comparison. Both controls must reach the relevant method and +fail at the returned-action lookup assertion. The nullish scope does not imply +ordered, range, `IN`, live-query, bucket-identity, or instrumentation coverage. + +## Execution and hostile review + +The reviewed code head is `0d21a22ec56bc06a47776690d7cdbfded0ae9acc`. +The five ledger blocks were removed at that head; no other test block in either +source file was retired. The oracle and both remaining source files passed +411/411 targeted tests with zero type errors. The named replay manifest's +selector tests passed. Fixed seed `20261012` and a seedless campaign each run +the same nullish property with 100 histories per index type. Guarded replay at +seed `20261012`, path `0` selected only the BasicIndex and BTreeIndex random +lanes and recorded one completed 100-run witness for each. + +The in-test skipped-update and unioned-lookup controls fail at the intended +Map-versus-lookup assertion. For replay failure fidelity, a temporary wrong +expected rule treated every `undefined` bucket as empty. It failed the +`equalityLookup(undefined)` comparison with seed `20261012`, shrink path +`0:1:1:1:1:1:0`, and counterexample `put(a, undefined)`. The guarded replay +reached one BasicIndex witness, marked it failed, and exited nonzero. After +restoring the oracle, that exact replay passed. This was a temporary model +fault for replay calibration, not a production mutant or a production bug. + +The independent hostile reviewer `/root/review_nullish_membership` first found +that the new property lacked a checked replay manifest entry. That setup +failure was not counted as an oracle kill. The manifest entry, selector, and +async property reporter were repaired. The reviewer rechecked the committed +head after the commit hook and found no remaining semantic blocker. No +production defect was demonstrated, so production code did not change. + +## Oracle guide audit at the reviewed head + +| Requirement | Outcome | +| --- | --- | +| ORC-001 | The established direct-index tests and `IndexInterface` exact-lookup methods supply the law. The claim excludes live-query WHERE nullish equality and the other observations listed above. | +| ORC-002 | A native source Map computes exact buckets without production normalization, comparator classification, or index storage. | +| ORC-003 | Opening prose states the law and limits; adjacent prose names the Map rule, legal actions, real index methods, observed sets, and returned-action checkpoint. | +| ORC-004 | Five fresh histories reconstruct the old witnesses; the adjacent `null→undefined` case and generated put/delete histories add nearby states. Removing either nullish value loses the distinguishing law; removing defined values loses defined/nullish moves; one key loses surviving-peer histories. The domain is three keys, four values, and at most 40 generated actions. An absent-key direct `update` is excluded because the source driver calls `add`. | +| ORC-005 | Both index types call real `add`, `update`, and `remove`, then compare exact returned sets and key count after each action. The fixed histories reach all five source states. | +| ORC-006 | A skipped `undefined→null` update and a unioned `lookup('eq')` result each cause an assertion failure at the exact-membership check. | +| ORC-007 | Fixed and seedless campaigns share one property, generator, recorder, comparison, and 100-run budget. The named guarded replay reaches the requested seed/path, and a captured controlled failure replays as a failed witness. | +| ORC-008 | The Map keeps key/value ownership because a later update or delete can distinguish sources with equal bucket sizes but different owners. | +| ORC-009 | `put` is a model source action mapped to index `add` or `update`; `delete` maps to `remove`. The shared terms `history`, `observation`, and `checkpoint` follow the glossary. | +| ORC-010 | The wrong-result spies restore in `finally`; the replay guard retains the failure witness separately from process status. No external resources are acquired. | +| ORC-011 | No plausible shared semantic fault requiring a second formulation was identified for the bounded nullish Map rule. Ordered comparator behavior is deliberately outside it. | +| ORC-012 | This record names the exact reviewed code head, audit result, controls, execution, and remaining scope. | +| ORC-013 | The `undefined→null` singleton and coexisting null/undefined adds distinguish no-op update and bucket-union alternatives at returned-action lookup checks. | +| ORC-014 | No controlled provider or host premise supports a claim about a real provider or host. | + +The next nullish candidates need their own observations: ordered and cursor +reads, `IN` union, custom comparator arguments, and the public live-query +path. The remaining source tests continue to own their exact assertions until +those transfers pass the same gates. diff --git a/docs/contributing/oracle-reviews/index-unit-retirement-round-3.md b/docs/contributing/oracle-reviews/index-unit-retirement-round-3.md new file mode 100644 index 0000000000..93d4543428 --- /dev/null +++ b/docs/contributing/oracle-reviews/index-unit-retirement-round-3.md @@ -0,0 +1,97 @@ +# Index unit retirement: exact and IN membership + +The source-isolated Field Lab loss audits used `b56d1a45c` as the frozen +baseline. The receiving owner is +`packages/db/tests/index-update-oracle.property.test.ts`. Its prior nullish +Map law checked `equalityLookup` and `lookup('eq')`, but explicitly excluded +`IN`. The four legacy blocks below were retired at reviewed code head +`d880f3139bbbba7141dc658d84d3602217c2960f`. + +## Transfer ledger before retirement + +| Legacy block in `btree-index-undefined-values.test.ts` | Smallest source history and returned-add observation | Plausible wrong result | Receiving witness | +| --- | --- | --- | --- | +| `should find items with undefined indexed value` | Add `a=active`, `b=undefined`, `c=inactive`, `d=undefined` on `status`; `equalityLookup(undefined)={b,d}`. | A lookup drops one of the two owners or includes a string peer. | The fixed four-add string history computes every exact set from a source Map after each add. | +| `should return empty set when no undefined values exist` | Add `a=1`, `b=2`; `equalityLookup(undefined)=∅` in a nonempty index. | A missing-bucket fallback returns a defined key. | The fixed all-defined history reads the absent group after both adds. | +| `should handle undefined in inArrayLookup` | Add `undef=undefined`, `one=1`, `two=2`, `three=3`; `inArrayLookup([undefined,1])={undef,one}`. | The method ignores the undefined operand, returns only one group, or includes a distractor. | The fixed four-add history and mixed-list probe compare the full set from the Map. | +| `should handle inArrayLookup with only undefined` | Add `undef1=undefined`, `undef2=undefined`, `one=1`; `inArrayLookup([undefined])={undef1,undef2}`. | An undefined-only list returns empty or loses one owner. | The fixed three-add history and singleton-list probe compare the full set from the Map. | + +The `IndexInterface` exposes direct exact and `IN` methods. A source Map owns +keys and values; equality selects one exact-value group, while `IN` selects +keys whose source value occurs in its operand list. The expectation does not +call either production lookup, normalized-key helper, or comparator. The +driver checks both direct methods and `lookup('eq')`/`lookup('in')` for +BasicIndex and BTreeIndex after each source action returns. A generated +three-key, five-value put/delete grammar checks nearby transitions with empty, +repeated, both orders of a mixed list, and nullish-only operand lists. The +fixed four-key and string histories preserve distinctions outside that domain. + +This is a direct-index set-selection law. It does not specify live-query WHERE +or `IN` nullish semantics, ordered traversal, range lookup, bucket identity, +or instrumentation. The `index-update-short-circuit.test.ts` block that also +calls `lookup('in')` stays: its `performance.now` non-call assertion cannot be +observed by this result oracle. The BTree file's traversal, range, custom +comparator, storage, and live-query blocks also stay. + +## Execution and hostile review + +The four ledger blocks were removed; the BTree file now has 12 tests instead +of 16. The receiving oracle, the remaining BTree file, and the short-circuit +file passed together: 423/423 tests and no type errors. The receiving oracle's +fixed-seed and seedless campaigns use the same 100-run property for each index +type. The guarded replay for `index-update.nullish-membership` at seed +`20261012`, path `0` reached two executions, one per index type, with 100 runs +each. The named replay manifest checks passed. + +The in-test wrong-result controls make a lookup lose one undefined owner, +drop the trailing undefined operand in `[1, undefined]`, lose one owner from +`[undefined]`, and return only the first group through `lookup('in')`. Each +reaches the relevant method and fails the Map comparison. The earlier +skipped-update and unioned-equality controls still pass their calibration. + +For failure replay, a temporary wrong expected rule cleared the Map result +for `[undefined, 1]`. The random lane failed at the direct `inArrayLookup` +comparison on `put(a, undefined)` with seed `20261012` and shrink path +`0:1:0:1:1:1:1:0`. Guarded replay reached both index lanes, recorded a failed +one-run witness for each, and exited nonzero. After removing the temporary +fault, that exact replay passed both lanes. This was a model fault for checker +and replay calibration, not a production mutant or a production defect. + +The fresh hostile reviewer `/root/review_in_membership` inspected the source +blocks, production implementations, ledger, oracle, and coverage map. It found +a position-sensitive survivor: all original mixed probes had undefined first. +Adding `[1, undefined]` and a control that drops its trailing undefined +operand killed that survivor. The reviewer independently ran the nullish +membership slice (36 passed, no type errors) and found no remaining semantic +blocker to retiring the four named blocks. It did not run a production mutant. +No production change was justified by this round. + +The reviewer rechecked committed head +`d880f3139bbbba7141dc658d84d3602217c2960f` after the commit hook. The +commit removed exactly the four ledger blocks and retained the reviewed oracle +and coverage claims. The reviewer reran all three targeted files: 423/423 +passed with no type errors. `git show --check` was clean. No new blocker arose. + +## Oracle guide audit + +| Requirement | Outcome | +| --- | --- | +| ORC-001 | `IndexInterface` and the established direct-index tests supply exact and `IN` set-selection obligations. The claim excludes WHERE semantics, ordered and range reads, bucket identity, timing, and arbitrary JavaScript values. | +| ORC-002 | A source Map supplies ownership. Expected exact sets come from Map grouping; expected `IN` sets come from operand membership over source rows. Neither uses production normalization, comparator classification, or another production lookup. | +| ORC-003 | Prose beside the model names the law, grammar, direct index driver, returned sets, and after-action checkpoint. | +| ORC-004 | The four fixed histories reconstruct every retired source state and operand. The generated grammar has three keys, five numeric/nullish values, up to 40 put/delete actions, and empty, repeated, mixed, reversed-mixed, and nullish-only lists. The fixed string and four-key histories cover values and width outside that grammar. Removing undefined loses the reported case; removing null loses nullish distinction; removing defined values loses mixed selection and moves; one key loses peer retention. Direct update of an absent source key is excluded. | +| ORC-005 | Both real index classes execute `add`, `update`, and `remove`. The checker compares `equalityLookup`, `lookup('eq')`, `inArrayLookup`, `lookup('in')`, key count, and indexed keys after each action returns. | +| ORC-006 | Four new wrong-result controls fail at the intended comparison; the temporary wrong expected rule also fails the generated `IN` comparison. No production mutant was run. | +| ORC-007 | Fixed and seedless campaigns share one generator and check with 100 runs each per type. Guarded seed/path `0` replay reached both lanes; the captured `0:1:0:1:1:1:1:0` failure replayed as failed, then passed after restoration. | +| ORC-008 | The Map retains each key's value because a later update or delete can distinguish states with the same bucket sizes but different owners. No model state was removed. | +| ORC-009 | `put` maps to index `add` or `update` according to source ownership; `delete` maps to `remove`. `IN` operands are lists of source values, and the after-action comparison is the checkpoint. | +| ORC-010 | Wrong-result spies restore in `finally`. Guarded replay records the property failure separately from child exit and cleans its temporary witness channel. The temporary model fault was removed after replay. | +| ORC-011 | No plausible shared semantic fault requiring a second independent formulation was identified for this bounded primitive-value Map rule. The review's order-sensitive survivor was an enforcement gap and is now checked. | +| ORC-012 | This versioned ledger records source assertions, law limits, execution, hostile review, and remaining scope against exact reviewed code head `d880f3139bbbba7141dc658d84d3602217c2960f`. | +| ORC-013 | `[undefined, 1]` versus `[1, undefined]` distinguishes operand-order dependence; `[undefined]` with two owners rejects first-owner-only selection; `[]` and `[1, 1]` check empty and duplicate boundaries. The wrong-result controls fail at returned-set comparisons. | +| ORC-014 | No controlled provider or host premise is used to claim real-provider or host behavior. | + +The remaining legacy blocks keep their distinct observations. The +short-circuit timing assertion, BTree traversal and range behavior, custom +comparator arguments, storage representation, and public live-query result +still need their own owners before retirement. diff --git a/docs/contributing/oracle-reviews/index-unit-retirement-round-4.md b/docs/contributing/oracle-reviews/index-unit-retirement-round-4.md new file mode 100644 index 0000000000..114c47c58e --- /dev/null +++ b/docs/contributing/oracle-reviews/index-unit-retirement-round-4.md @@ -0,0 +1,120 @@ +# Index unit retirement: undefined ordered reads and ranges + +The source-isolated Field Lab loss audits compared the remaining direct +traversal and range blocks in `btree-index-undefined-values.test.ts` against +`index-update-oracle.property.test.ts` at frozen head +`df18f41bba07589c4aa8a4b17ca676be7d39dc15`. The receiver checked one-owner +undefined cursors and numeric ranges, but its generated nullish lane checked +only exact and `IN` membership. The table freezes the source assertions before +changing the receiver. A proposed witness is not retirement evidence until it +executes, rejects a wrong result, and survives hostile review. + +| Legacy block | Source assertion at the returned-action checkpoint | Loss in frozen receiver | Proposed transfer | +| --- | --- | --- | --- | +| `take(n, undefined)` versus `takeFromStart(n)` | After `a=undefined,b=1,c=2`, `takeFromStart(3)` has three keys and `take(3, undefined)=[b,c]`. | The explicit cursor has a singleton analogue; the default-policy endpoint cut is absent. | Sort source rows with nullish values first; compare both exact arrays after each add, with a fixed three-row history. | +| `takeReversed(n, undefined)` versus `takeReversedFromEnd(n)` | In a fresh copy of the same state, the endpoint returns three keys and the explicit cursor returns none. | The explicit reverse cursor has a singleton analogue; the nullish endpoint cut is absent. | Compare both exact reverse arrays in the same fixed history. | +| Store and retrieve three undefined owners | After `a,b,c=undefined`, `takeFromStart(10)=[a,b,c]` and `equalityLookup(undefined).size=3`. | Two-owner exact membership is fixed; three-owner ordered output is absent. | Pin all three owners and compare exact arrays and source-derived equality sets. | +| Remove one of two undefined owners | After adding `a,b=undefined` and removing `a`, exact lookup has size one and contains `b`. | Reachable in generated put/delete histories, but the two-owner deletion cut is not pinned. | Pin the removal and compare the complete exact set after it returns. | +| Iterate past multiple undefined owners | After `undef1,undef2,undef3=undefined`, then `one=1,two=2`, `take(10,undefined)=[one,two]`. | One-owner cursor and multi-owner membership checks never meet. | Pin the five-row history and compare exact cursor arrays after each add. | +| Explicit undefined versus omitted lower range bound | After `undef=undefined,one=1,two=2,three=3`, `{from:undefined,to:2}` inclusive returns exactly `{undef,one,two}`; `{to:2}` returns three keys. | Numeric bounds and nullish upper-only reads are separate fixtures. | Source-derived range sets check explicit and omitted lower bounds, plus exclusive variants that distinguish them, after a fixed four-row history. | +| Double undefined range bound | After `a,b=undefined,c=1`, `{from:undefined,to:undefined}` inclusive returns exactly `{a,b}` and excludes `c`. | No double-bound range check with two undefined owners. | Pin this state and compare the full set against source-derived bounds. | + +The proposed law follows `DEFAULT_COMPARE_OPTIONS` and `defaultComparator` +in `packages/db/src/utils.ts` and `packages/db/src/utils/comparison.ts`, the +direct cursor comments in `packages/db/src/indexes/btree-index.ts`, the +key tie-order contract exercised by `deterministic-ordering.test.ts`, and the +frozen legacy assertions. `IndexInterface` establishes the direct methods but +its reference page does not specify these ordering details. The bounded law +uses default ascending order with nullish values first, positive numbers +ascending, and row keys ordered within one comparator position. Its domain is +`undefined`, `null`, and the numbers 1–3; negative numbers are outside it. +`null` and `undefined` occupy one comparator position for ordered reads while +remaining distinct exact lookup groups. A source Map retains key ownership. +The model sorts and filters that Map; it does not call the index comparator, +normalization, cursor, or range machinery to compute expected results. +`take` and `takeReversed` exclude the cursor's comparator position. Endpoint +methods have no cursor. A range bound exists only when its field is supplied; +an inclusivity flag without that bound has no effect. The production path is +the direct `IndexInterface` on both BasicIndex and BTreeIndex. Unfiltered +ordered arrays at n=1, 3, and 10 and full sets for ten selected forward range +shapes are compared after each add, update, or remove returns. The owner does +not claim all bound combinations or filtered takes. + +The default-order and string-order tests inspect `orderedEntriesArray`, a test +view of internal entries. Those two blocks remain until their private +observation has a receiving owner or the test's representation claim is +explicitly retired. The custom-comparator argument, `valueMapData` storage, +and public live-query blocks also remain. This direct-index law makes no claim +about those observations or live-query scheduling. + +## Execution and hostile review + +The seven ledger blocks were removed after the hostile review. The BTree file +now has five tests. The receiving oracle and that file passed together: +415/415 tests. The package TypeScript check and `git diff --check` passed. +The fixed-seed and seedless nullish campaigns run the same source actions, +checker, and 100-run budget for both index classes. A guarded replay at seed +`20261012`, path `0`, reached both classes with 100 successful runs each. The +named replay harness passed its 36 tests. + +Five wrong-result controls force an endpoint read to lose the undefined owner, +an explicit cursor to repeat one of three undefined owners, an inclusive +explicit lower bound to drop undefined, a double undefined range to lose a +peer, and a null bound to drop its undefined comparator peer. Each fails the +intended source-Map comparison for both index classes and restores its spy in +`finally`. The earlier exact and `IN` controls continue to run. + +For failure replay, a temporary wrong model rule cleared the expected set for +an explicit `undefined` lower bound and upper bound 2. The fixed campaign +failed at the forward range comparison on `put(a, undefined)`, seed `20261012`, +shrink path `0:0:0:0`. Guarded replay at that seed and path recorded one failed +run for each index class and exited nonzero. After removing the temporary rule, +the same guarded replay passed both classes. This was checker calibration, not +a production mutant or product defect. No production code changed. + +The fresh hostile reviewer `/root/review_ordered_range_round4` read the full +legacy file, ledger, changed oracle, coverage map, reference material, and both +index implementations. It identified a surviving wrong design in the broader +claim: a range implementation could treat a `null` bound as an exact-null +bucket because no probe used null as a bound. Inclusive and exclusive null +probes plus a wrong-result control closed that gap. It also required the +contract authority and domain limits to be precise. Its final block-by-block +verdict found no lost source assertion among the seven retired blocks. It did +not run a production mutant. The default-order internal-entry, string-order +internal-entry, custom-comparator argument, storage-view, and live-query tests +remain in the source file. + +## Oracle guide audit + +| Requirement | Evidence and limit | +| --- | --- | +| ORC-001 | The default comparator, BTree cursor comments, deterministic key tie ordering, and frozen direct tests support the bounded law. The interface supplies the methods, not the full semantics. The claim is limited to default options, values `undefined`/`null`/1–3, unfiltered takes at n=1/3/10, and ten selected forward range shapes. | +| ORC-002 | A source Map owns rows. A fresh value-position sort predicts arrays; separate model-bound objects filter source rows for ranges. Neither calls production normalization, comparator, or traversal to compute an expected result. | +| ORC-003 | Opening and adjacent prose distinguish the law, source Map, action grammar, direct index driver, returned arrays/sets, and after-action comparison. | +| ORC-004 | Fixed three-row endpoint, five-row repeated-owner, two-owner removal, four-row range, and double-bound histories reconstruct all retired assertions. The generated put/delete tail has three owners and five values over 1–40 actions. Removing an endpoint method loses the absent-cursor distinction; one owner loses peer traversal; removing null loses comparator-position overlap; removing numeric successors loses cursor continuation; omitting bound presence loses the exclusive comparison. Negative numbers, arbitrary types, filtered takes, reverse ranges, and direct update of an absent owner are excluded. | +| ORC-005 | BasicIndex and BTreeIndex direct methods run after returned add, update, or remove. Exact arrays and complete sets preserve omissions, duplicates in arrays, order, and wrong membership. | +| ORC-006 | Five injected wrong results fail at the intended comparisons. The temporary wrong model rule failed at the forward range comparison and was removed. | +| ORC-007 | Fixed and seedless lanes use the same generated property and 100-run budget. Guarded seed/path `0` replay reached both lanes; captured failure path `0:0:0:0` failed both lanes under the temporary rule and passed after restoration. | +| ORC-008 | No model state was added: the existing Map retains owner-to-value association, needed for later update and delete. Sorted rows and bounds are recomputed observations. | +| ORC-009 | A model `put` maps to add or update according to source ownership; `delete` maps to remove when owned. Model comparator position combines null and undefined only for order/range, while exact membership keeps them distinct. | +| ORC-010 | Spies restore in `finally`; the guarded runner recorded the primary mismatch separately from process exit and removed its private witness channel. | +| ORC-011 | The reviewer found a missing probe, not a fault shared by the model and production. The added null-bound probe distinguishes the model rule from an exact-bucket interpretation. | +| ORC-012 | This versioned ledger holds source assertions, transfer, limits, execution, hostile verdict, and the exact committed-head recheck below. | +| ORC-013 | Explicit inclusive versus exclusive undefined and null bounds, paired with omitted bounds, distinguish presence and comparator-position grouping. The wrong-result controls fail at those comparisons. | +| ORC-014 | No host or external provider behavior is claimed. | + +The public live-query issue regression remains the only test in this file that +crosses the Collection/query path. The other four retained tests observe +private BTree ordering or storage and custom-comparator arguments; direct +source-Map arrays and sets cannot replace those observations without a +separate contract decision and receiving witness. + +## Exact committed-head recheck + +The same hostile reviewer rechecked committed code head +`a72a7043d5750b33100ab95b7d3c6a490b15626c` after the commit hook. The +commit deletes exactly the seven reviewed blocks and keeps all five intended +legacy tests. The committed oracle retains the Map comparisons, null-bound +probes, and five wrong-result controls; no temporary calibration fault remains. +The two targeted files passed 415/415 tests with no type errors, and +`git show --check --oneline HEAD` passed. The reviewer found no new blocker. diff --git a/docs/contributing/oracle-reviews/pr-2116-confirmed-fixes.md b/docs/contributing/oracle-reviews/pr-2116-confirmed-fixes.md new file mode 100644 index 0000000000..676213d656 --- /dev/null +++ b/docs/contributing/oracle-reviews/pr-2116-confirmed-fixes.md @@ -0,0 +1,186 @@ +# PR #2116: confirmed fixes found during unit-test retirement + +The seven index, five non-finite comparison, two ordinary Temporal comparison, +and six Date/RegExp assertions retired by this PR have their own loss ledgers in +the [coverage map](../oracle-coverage.md#unit-test-retirement-transfers-in-pr-2116). +Those transfers did not authorize new product guarantees. Each additional fix +below has an accepted law, a failing observation before repair, an independent +receiving witness, and a bounded hostile challenge. `origin/main` was +`bbbf017ef5b3884515295a4225faacdfd814499e` for the baseline checks. + +| Law and authority | Baseline failure | Receiving observation and hostile result | Limit | +| --- | --- | --- | --- | +| A full BasicIndex build keeps `undefined` at the configured nulls-first position. The default comparator and surviving ordered-index tests supply the rule. | Native `Array.sort` moved a raw `undefined` value last without calling the comparator. | The source-Map direct-index owner compares complete ordered reads after full build. The [build review](basic-index-nullish-build.md) records RED/GREEN and hostile evidence. | Default nullish order and bounded build histories. | +| Duration identity preserves all ten public fields. Changelog #1519 explicitly keeps equivalent displayed forms unequal for query equality; the Collection subscriber-mirror contract requires a changed row to reach its subscriber. | One second and 1000 milliseconds display alike. The former `deepEquals` and normalized index key treated them as equal, suppressing a Collection change message or returning a wrong exact match. The original implementation failed direct BasicIndex/BTreeIndex lookup, public scan and indexed `eq`, and constant and row-supplied `in` probes. | The deep-equality and change-event oracles compare a direct result, stored row, callback batch, and subscriber mirror. The Duration query owner checks ten fields through BasicIndex/BTreeIndex exact lookup and public `eq`, while the `in` evaluator owner uses constant and row-supplied lists. A seconds-only production mutant fails the adjacent-unit deep-equality property. Executed wrong-key controls reject display-string, sign-loss, and first-nonzero-field query keys; they were not production mutants. | Fixed replacement and single-row query histories; no arbitrary multirow update or live-query publication claim. | +| An indexed ZonedDateTime `orderBy` follows instant order, as promised by changelog #1519. A time-only Duration uses its elapsed-time order while keeping exact field identity. | Indexed first-row reads placed a later ZonedDateTime first when display text opposed epoch order, including a 9ns/10ns digit-width pair. PT120M also preceded PT1H under string-key order. | The Temporal index-order oracle compares exact direct BasicIndex/BTreeIndex reads and bounded public first rows, including an on-demand Collection. Adjacent nanoseconds at a large epoch reject `Number` precision loss. PT1H/PT60M exclusive cursors reject an exact-suffix tie-break; negative durations reject absolute-value order. The hostile reviewer found no sign mismatch in 2,000 seeded valid day/time pairs. | ZonedDateTime instant and time-only Duration order in the named fixed histories. Calendar units needing `relativeTo`, mixed Temporal kinds, custom index callbacks, and arbitrary update histories remain open. | +| A manual index cannot change the declared ascending `orderBy(rank).limit(1)` answer. Query order is public; index selection is physical planning. | A reverse manual BTree index made `currentStateAsChanges` return rank 30 instead of rank 10. | The custom-index selection oracle independently sorts two distinct numeric ranks and checks the direct public snapshot and an eager live-query result. The hostile reviewer found the direct path that the first witness missed; both paths now pass. An added compiler guard had no demonstrated effect on this witness and was removed. | Eager initial reads with distinct ranks; no callback-tie, on-demand, later-update, or work-bound claim. | +| Index updates conserve each source key even when number and string values occupy different comparison domains. | The old comparator could make `2` tie `"2"` while putting them on opposite sides of `"10"`; a direct deletion then duplicated or lost a key. | A source Map predicts full direct traversals after bounded delete/update/re-add actions, and a bounded indexed Collection read checks the public key set. The hostile reviewer confirmed the original comparator fails the source-Map comparison. | Key conservation, not a mixed-kind public order policy or all Collection histories. | +| An ordinary string beginning `\0tanstack-db:` obeys its declared string collation in indexed and scan reads. Internal Temporal order keys must not cause a tree to lose a neighboring ordinary string. | On `origin/main`, an en-US locale scan selected `\0tanstack-db:a` first while BasicIndex/BTreeIndex selected `tanstack-db:r`. Before the final kind classifier, deleting a neighboring ZonedDateTime or Duration also made BTreeIndex omit the surviving string. | The direct-index owner compares first public rows through scan and indexed paths under locale and custom string collations, with row IDs independent of label order. Two three-key BTree histories compare all surviving keys. The hostile reviewer ran 24 focused tests, 108 add/delete permutations, and a 27-value comparator transitivity sweep across five option variants. After the final classifier simplification, it also ran 3,456 four-key BasicIndex/BTreeIndex insertion/deletion histories (27,648 checkpoints) and a 16-value comparator sweep; no in-domain counterexample remained. | Named scalar, binary, Temporal, and string-kind histories. No arbitrary nested values, mixed-kind relative order, or arbitrary custom index `compareFn` guarantee. A separate two-value direct callback lane checks legacy scalar input kinds for immutable Date, Temporal.Instant, and Uint8Array values. | + +## Design decisions and open work + +The removed `subset-dedupe.test.ts` case that overrode `Date.getTime` and +`Uint8Array.slice` was omitted from the earlier retirement ledger. Its frozen +assertion checked that request transport kept the same `where` object and that +the evaluator produced the same answers before and after transport. A probe on +this PR still passes that case. The maintainer excluded guarantees after callers +modify standard data objects, so the overridden-method case has no receiving +oracle law. Immutable Date, byte-array, and Buffer request identity and matching +remain checked in `subset-dedupe.test.ts`; `immutable-demand-boundary.test.ts` +also checks the direct and deferred request handoff. This records the excluded +assertion explicitly rather than treating its deletion as a transfer. +The rewritten `binary-equality-work.test.ts` also stopped editing a compiled +byte literal in place. The evaluator still passes that old trace on this PR, +but in-place edits to standard values are outside the accepted contract. Its +fresh-value checks remain the supported equality and work witnesses. + +The captured-`inArray` list rebuild and per-row custom index callback ordering +were proposed extensions, then withdrawn by the maintainer. This PR does not +promise behavior after callers modify standard data objects in place, unsigned +byte ordering, or special handling for malformed Temporal lookalikes. The +[earlier-findings record](pr-2116-deferred-findings.md) keeps Temporal admission +observations, mixed-kind query order, the confirmed pre-existing indexed +Temporal exception gap, and earlier-branch observations separate from the +confirmed fixes above. + +The repeatable workflow is: freeze source assertions in a loss ledger; name +the receiving law and its limits; obtain a baseline failure before production +changes; extend an independent model at the public checkpoint; ask a fresh +hostile reviewer to attack authority, reach, and plausible mutants; then +recheck the exact committed head. A production-code budget is evaluated +separately from oracle and documentation growth. A green generated campaign +supports only its legal histories, paths, and observations. + +## Production-code weight + +Against the recorded `origin/main` baseline, the source diff under +`packages/db/src` adds 356 lines and removes 535, for **179 fewer production +lines**. The built DB ESM entry bundled with esbuild (`bundle`, `minify`, +`format: esm`, `platform: neutral`, `target: es2020`, +`NODE_ENV=production`; DB IVM, pacer-lite, and standard-schema external) is +335,613 bytes / 100,492 gzip at baseline and 336,504 bytes / 100,889 gzip at +executable head `905b93323`. That is **891 more minified bytes / 397 more gzip +bytes**. Both outputs use Node's `zlib.gzipSync`. The source-line budget is +negative; the consumer-bundle budget remains positive. + +The later build-cache commit `ad6f22105` removes the obsolete `compareStored` +minifier entry and registers 23 private members that the cache check recognizes +as safe. Production TypeScript is unchanged from the reviewed executable head. +After a fresh build, the same consumer-bundle measurement is 335,182 bytes / +100,737 gzip: **431 fewer minified bytes / 245 more gzip bytes** than baseline. +The local cache check and minified public API check pass. This narrows, but does +not close, the consumer-bundle gap. The repository's separate compressed-size +check reports **395 fewer bytes** across its published ESM package files at +`ad6f22105`; that unbundled package measure is net negative. + +## Exact-head hostile review of the index rewrite + +The executable revision is `905b93323` (following `2f852dcd1`); the baseline +is `origin/main` `bbbf017ef5b3884515295a4225faacdfd814499e`. The rewrite +replaces BTree's per-exact ordered entries with comparator groups while keeping +exact membership separate. Its binary ownership history compares a source Map +with complete forward and reverse index reads. An intermediate PR rewrite lost +a surviving byte owner after a fresh equal-content removal. The published +parent `78bd4946b` passes the six final binary cases, so that particular RED +is **not** evidence of a pre-existing defect. + +The hostile reviewer ran two wrong-output controls against the rewritten +BTree. One suppressed a surviving short-byte key after equal-content removal; +the oracle failed at the ordered-read length comparison. A second suppressed +one key after inserting two 8193-byte values that differ only in their last +byte. That mutant survived the first version of the binary lane because the +wide-byte case checked exact lookup only. The final oracle adds full forward +and reverse key conservation there; the same mutant then failed at that +comparison. The unmodified six cases passed. These controls are output +mutants, not claims that `origin/main` failed. + +The final BasicIndex retains the original stable byte value in its sorted +array and reuses the normalization cache populated at admission. A direct +offset-view and Buffer probe preserved exact membership and ordered keys after +fresh equal-content removal. Callers who edit the retained byte array in +place have no promise. This removes a redundant byte copy without changing +the accepted immutable-value law. Duplicate-heavy insert and removal timings +were near parity in the bounded A/B probe. An attempted merge of Array and +ordinary-object index kinds was discarded: it produced a strict three-value +comparator cycle, so the separate kind remains. + +The fresh review also reconstructed a public scan/index mismatch for mixed +`PlainDate`/`PlainDateTime` and for two `PlainMonthDay` values. The scan throws +as changelog #1519 requires, while either index returns rows. It reproduces +on clean `origin/main` and this head. The [deferred record](pr-2116-deferred-findings.md) +assigns the exception witness and ordered-query repair to the Temporal owner; +the bounded ZonedDateTime and time-only Duration claims above do not include +this case. + +| Guide requirement | Exact-head result | +| --- | --- | +| ORC-001 contract and limits | Transfer ledgers trace the retired assertions; changelog #1519 and the surviving index interface give the separate Temporal and key-conservation laws. The coverage map names excluded mixed-kind order, mutation of standard values, and wider histories. | +| ORC-002 independent judgment | Source Maps, supplied Duration fields, and supplied Temporal instants predict results without importing index grouping, normalization, or comparator classifications. | +| ORC-003 literate responsibilities | The owners state each law before mechanics and explain the model, legal actions, direct or public production path, observation, and checkpoint beside the code. | +| ORC-004 generated grammar | The transfer ledgers account for reconstruction, ablation, bounded ranges, and excluded histories of the existing generated index lanes. The new binary lane is a fixed history, so the generated-grammar trigger does not apply to that lane. | +| ORC-005 path and observation | Complete returned arrays expose missing or duplicate keys in both directions; exact lookups expose byte-content errors. Collection reads and index spies establish the named public routes. No earlier-publication information claim is made. | +| ORC-006 checker calibration | Short-byte and 8193-byte wrong-output mutants failed at ordered-read assertions; the wide mutant first survived, motivating the final assertion. Existing transfer ledgers retain the other hostile controls. | +| ORC-007 fixed and random replay | The important generated index properties retain their documented fixed and seedless campaigns and direct seed/path replay. The fixed binary examples do not trigger this requirement. | +| ORC-008 model state | The source Map retains key ownership because a later delete or replacement can distinguish live rows; the binary fixed history adds no production-shaped index state. | +| ORC-009 vocabulary | Model puts, deletes, source keys, and ordered reads map to direct index actions and public observations. Comparator groups remain production storage, not reference-model state. | +| ORC-010 failure and cleanup | Existing generated drivers retain their replay context and failure cause; public Collection witnesses clean up in `finally`. The fixed direct binary cases acquire no external resource. | +| ORC-011 second formulation | No plausible fault shared by the source Map and index ownership model was identified inside the bounded transfer. The separately confirmed mixed-Temporal public error is recorded as an unresolved neighboring law, rather than inferred from direct index equality. | +| ORC-012 review record | This versioned record identifies the executable head, baseline, mutant outcomes, tests, size, and unresolved public path. | +| ORC-013 boundary witness | Nullish boundary controls, opposing 9ns/10ns display and instant order, adjacent large-epoch nanoseconds, and Duration display ties reject nearby wrong rules at their named checkpoints. | +| ORC-014 controlled premise | The controlled Collection fixtures support only their named Collection observations; no real-provider behavior is claimed, so the cross-provider handoff trigger does not apply. | + +The DB source suite passed 13,192 tests in 282 files before the no-copy +follow-up. After that follow-up, the affected index and comparison files passed +623 tests, the built-package lane passed 291 tests, the minified DB check +passed, and the DB source TypeScript and affected-file lint checks passed. +Those runs cover the changed immutable-value paths; they do not establish +arbitrary later Collection histories or the deferred mixed-Temporal error. + +## 2026-10-13 high-effort review of PR head `16b219224` + +This addendum supersedes the earlier manual-index row's two-rank limit. The +earlier observations remain historical evidence. The follow-up tree has a +three-rank/two-row owner and one ordered-index eligibility check shared with +the mixed-kind repair. The reviewer withdrew the claimed wrong live-query +result after finding that its scratch test resolved `@tanstack/db-ivm` from a +different checkout. With the correct package, both the frozen PR head and +`origin/main` published the expected prefix before and after the reported +writes. + +| Review IDs | Law, experiment, and action | Remaining limit | +| --- | --- | --- | +| R01, R10 | Index choice cannot change an ordered public prefix. The new scan-versus-index owner rejected the frozen type-name order in eight initial ascending cases. Its repaired 20 ascending and descending BasicIndex/BTreeIndex cases include paired source updates and a ReverseIndex route. Forcing the index-eligibility getter true rejects all ten original ascending histories. A retained on-demand BTree history starts with numbers, inserts and moves a bigint, deletes a prefix row, then requests a later window. It asserts each settled public prefix against independently sorted provider rows. The finite request supplies a missing row; the later request uses a cursor after the retained index has become ineligible for new ordered plans. Withholding either requested row fails the corresponding public assertion. A single physical and query comparator is unsafe here: the query comparator does not form a transitive total order for all mixed JavaScript values. | The eager histories and this retained on-demand paging history pass. The withheld-row controls challenge the provider result, not production code. Other provider paging sequences and repeated domain changes remain untested; the retained loader still traverses the index after eligibility changes. | +| R03 | The reviewer withdrew the `[20,30]` public-result claim after fixing the scratch test's cross-checkout `@tanstack/db-ivm` resolution. The corrected three-row eager fixture returns `[10,20]` on both the frozen PR and `origin/main`; the reviewer also checked the insert and delete checkpoints. Our widened owner independently sorts three ranks and compares eager and prepopulated on-demand public rows after preload. The compiler's ordered-index eligibility check stays because mixed-kind finding R01 needs it; the R03-only planner assertion was removed. | No confirmed live-query correctness defect remains for this fixture. Incomplete-provider paging and later updates are outside this owner. | +| R04 | Direct index callbacks retain the legacy scalar input kinds for immutable Date, Temporal.Instant, and Uint8Array values. Five of six new callback cells failed on frozen head; callback-only normalization makes all six pass, with raw-object and collapsed-position hostile controls rejected. | Two-value direct-add histories only; arbitrary callbacks and later updates are not claimed. | +| R06, R07 | BasicIndex now keeps admitted exact keys beside ordered representatives; a 64-row Temporal range and ordered read made zero `toString()` calls, down from 128. BTreeIndex keeps a comparator-tie bucket map after one exact bucket leaves; a return to the tie no longer copies every peer. | BasicIndex may retain one former representative per live exact group. No general heap limit is claimed beyond existing retained-index state. | +| R08, R09 | Object comparison does allocate temporary reference-identity tuples, but a bounded timing probe found no object throughput regression; no extra hot-path code was added without a demonstrated cost. Auto-index now shares `indexableOperations`, and `DurationFields` is defined once. | An allocation profile under a representative workload would be needed to establish a material R08 performance defect. | +| R02, R05, R11 | In-place byte edits can stale the normalized-key cache, and the overridden-method request test was omitted from the earlier retirement record. Both concern caller modification of standard data objects, which the maintainer excluded; fresh immutable matching and request identity remain covered. The reviewer's random index run supplied no seed, grammar, or count. | The cache holds one key per live byte array. The random run is stress evidence, not a witness for mixed public order or callback inputs. | + +An adjacent previously recorded defect also changed under the mixed-domain +guard. On `origin/main`, ready BasicIndex and BTreeIndex direct ordered reads +returned a row for PlainDate beside PlainDateTime, or for two PlainMonthDays, +while the scan threw the `TypeError` promised by changelog #1519. The expanded +Temporal index-order owner now checks six scan/index cases at direct-read and +eager live-query checkpoints. All 68 tests in that file pass on this worktree. +The eager live-query paths already threw on `origin/main`; the repair evidence +is the indexed direct-read mismatch. On-demand and later-update exception +histories are not claimed. + +The same rewrite repairs a separate PlainDate boundary under lexical string +collation. The supplied year fields require `-100000-01-01` before +`+010000-01-01`, while their display strings have the reverse lexical order. +The expanded Temporal owner rejects the two `origin/main` indexed direct-read +results, which selected the positive year; scan and eager live-query preload +already selected the negative year. Current BasicIndex and BTreeIndex direct +reads pass. This establishes the fixed ready-source history, not every +extended-year pair or on-demand publication. + +The follow-up tree adds 406 and removes 541 production source lines against +`origin/main`, for **135 fewer lines**. A paired local minified build of the 105 +DB JavaScript package files totals 205,973 gzip bytes, versus 206,122 for the +same files on `origin/main`: **149 fewer gzip bytes**. Bundling each built DB +ESM entry into one consumer file with esbuild instead yields 100,746 versus +100,544 gzip bytes: **202 more gzip bytes**. Separate-file and whole-bundle +compression have different dictionaries and tree-shaking boundaries. React DB +has no source change in this follow-up; the CI size action has not yet run on +the final pushed commit. diff --git a/docs/contributing/oracle-reviews/pr-2116-deferred-findings.md b/docs/contributing/oracle-reviews/pr-2116-deferred-findings.md new file mode 100644 index 0000000000..ebb483c161 --- /dev/null +++ b/docs/contributing/oracle-reviews/pr-2116-deferred-findings.md @@ -0,0 +1,46 @@ +# PR #2116: earlier findings and guarantee decisions + +This record separates findings discovered during the unit-test retirement +workflow from the bounded fixes retained in PR #2116. The transfer ledgers +remain the authority for retired assertions. The +[confirmed-fixes review](pr-2116-confirmed-fixes.md) records the baseline and +hostile evidence for fixes that this PR now keeps. A finding below is not +closed by passing a neighboring transfer oracle. + +| Finding | Authority and evidence | Next owner and witness | +| --- | --- | --- | +| R29: two valid `Temporal.Duration` values can have the same display text but different fields. A changed field can be suppressed as an equal Collection value. | Closed in this PR for the tested direct and Collection paths. The Duration field model checks all ten fields; a same-display replacement and adjacent-unit histories make the former `deepEquals` and change-event behavior fail before the repair. The query equality owner and `in` evaluator owner separately check Duration field identity. | General multirow and live-query publication histories remain outside these owners. Add them to the relevant owner if a new failure reaches those paths. | +| R16: an indexed Collection can admit a valid Temporal row whose index comparator has no public order, then throw after partial state changes. | Earlier PR revisions recorded RED admission and publication traces for Duration, PlainMonthDay, and mixed-kind rows. A paired probe on `origin/main` `bbbf017ef5` and PR head `0c10ae8e8` found no failure: 190 valid Temporal pairs per index type and revision passed direct insertion; 24 two-row sync and optimistic Collection histories retained both public rows, exact memberships, ordered storage, ready status, and both insert publications at the second write return. The earlier RED traces establish an intermediate-branch failure, not a current-base or current-head defect. Public query comparison errors remain separate from internal index storage order. | No admission repair follows from this evidence. If a valid current-head admission history fails, extend the direct-index and Collection publication owners with exact membership, ordered storage, stored rows, and subscriber observations before changing production. The probe does not cover all valid Temporal values, native-host implementations, updates, builds, or longer histories. | +| Mixed-kind indexed order and custom-index query selection may disagree with public query results. | The source-Map owner closes the mixed number/string direct-index key loss. The new scan-versus-index owner rejected the frozen PR's wrong mixed-kind public prefixes, and the current guard makes the tested eager paths use the query's order. A retained on-demand BTree history checks visible prefixes after a number/bigint transition, a missing-row finite request, and a later cursor request that still traverses the index; all settled public windows match independently sorted provider rows. A reverse manual index produced a wrong direct prefix on an earlier branch revision; the current three-rank owner checks direct, eager, and prepopulated on-demand public results. The reviewer withdrew its claimed live-query failure after fixing a cross-checkout `@tanstack/db-ivm` resolution error. The maintainer withdrew a per-row custom callback position guarantee for comparator-equal exact values. | Other retained on-demand paging sequences, different incomplete-source patterns, and later live updates need public-result witnesses. The owners do not define a universal precedence among unlike kinds. | +| Indexed ordering can suppress a required Temporal comparison error. | Changelog #1519 requires `TypeError` for mixed or unordered Temporal values. On clean `origin/main`, indexed direct reads returned rows for `PlainDate` beside `PlainDateTime` and for two distinct `PlainMonthDay` values while the scan threw. The new Temporal exception lane fails on those four baseline indexed reads and passes on this branch; eager live queries already threw on main. | The bounded direct-read repair is complete for these pairs. On-demand acquisition, later updates, and other mixed Temporal pairs need separate witnesses before claiming the whole exception class. | +| R17: an equal-time Date removal left an ordered ghost under added index machinery. | The recorded RED trace depended on code introduced during this PR. It does not establish an `origin/main` defect. The stronger rule for a substituted old row with a different callback position was withdrawn. | If ordinary direct remove/update still has a current-base counterexample, add a source-Map history to the direct-index owner before changing production. | +| PlainDate extended-year order under lexical string collation can diverge between scans and indexes. | On `origin/main`, a ready scan returns `-100000-01-01` first while BasicIndex and BTreeIndex direct public reads return `+010000-01-01`. The new Temporal owner checks calendar order from the supplied years and fails on those two baseline direct reads. Both direct indexed reads pass on this branch. Baseline eager live queries already returned the right first row. | The fixed ready-source direct-read history is closed. Other extended-year pairs, later updates, and on-demand publication remain outside the current witness. | +| Cross-realm native equality was asserted by a surviving request-transport unit test. | PR #1797 captured the then-current `instanceof` result. No public realm-distinctness promise was found. The maintainer questioned that distinction. This PR removes that incidental test expectation and makes no cross-realm equality claim. | A future public contract decision needs a host-boundary witness before either equality policy becomes an oracle law. | + +The 2026-10-13 review expanded the custom-index owner to three ranks and a +two-row window. The reported `[20,30]` public result came from a scratch test +that resolved `@tanstack/db-ivm` from another checkout; with the correct +package, both frozen PR head `16b219224` and `origin/main` returned `[10,20]` +at the reviewer's read checkpoints. The reviewer withdrew that finding. A +separate paged-provider witness now reaches a retained +on-demand mixed-kind index after its domain changes. It checks public rows +after a finite request fills a missing prefix row and after a later cursor +request traverses the retained index. Wrong-provider controls that withhold +either requested row fail the public comparison. Other provider histories +remain untested; this witness does not establish general safe traversal of an +ineligible retained index. + +Captured `inArray` list-entry refresh and whole-live-query reconstruction were +explored as new features. The pre-PR evaluator and live graph did not already +deliver the whole-result behavior. The maintainer withdrew that expansion for +this PR because of its production-code and query-planning cost. A separate +proposal must define its dependency and invalidation boundary before adding +an oracle or production graph machinery. In-place edits to a Date or byte +item, unsigned default byte ordering, and guarantees after callers alter +standard objects are also outside the accepted contract. + +Earlier experiments and RED/GREEN records remain in published commits on +`codex/oracle-test-retirement-2-blocks`. This PR restores the confirmed, +bounded fixes described above. The R16 and R17 rows still distinguish +earlier-branch observations from a current-base failure; neither justifies +new production machinery without a baseline witness under the accepted law. diff --git a/docs/contributing/oracle-reviews/utils-unit-retirement-round-7.md b/docs/contributing/oracle-reviews/utils-unit-retirement-round-7.md new file mode 100644 index 0000000000..5bdfa6d9b0 --- /dev/null +++ b/docs/contributing/oracle-reviews/utils-unit-retirement-round-7.md @@ -0,0 +1,116 @@ +# Deep equality unit retirement: Date and RegExp + +Two source-isolated Field Lab loss audits compared the Date and RegExp blocks +of `packages/db/tests/utils.test.ts` at frozen head `1265abeb5` with the +existing `utils-oracle.property.test.ts`. The proposed deletion said that the +oracle's copy/change and cross-type laws already covered both blocks. The +audits found three missing primitive boundaries before either block was +removed. No production behavior was changed for this transfer. + +| Retired block and source observation | Frozen receiver | Receiving law and checkpoint | +| --- | --- | --- | +| Date: separately allocated 2023-01-01 Dates are equal; 2023-01-02 differs. | Date copy/change properties and a fixed Date-time triple covered the law, but not those exact operands. | The fixed 2023 triple reconstructs both results; the existing generated Date copy/change property checks neighboring timestamps at `deepEquals` return. | +| Date: a Date differs from its ISO string. | No Date/string pair; the broad cross-type description had compressed away this boundary. | A fixed 2023 probe and generated finite Dates compare with their own ISO text in both directions. | +| Date: a Date differs from its numeric timestamp. | No Date/number pair, even when the number encoded the same instant. | The same fixed and generated law compares with `getTime()` in both directions. | +| RegExp: separate `/abc/g` and `/test/i` copies are equal; changed flags or source differ. | Source/flags fixed triples and a generated copy/changed-flags property covered the law, but not those exact operands. | Fixed `/abc/g`, `/abc/i`, `/def/g`, and `/test/i` checks reconstruct each return; generated RegExps vary source and flags. | +| RegExp: `/abc/g` differs from its source string `abc`. | The receiver had RegExp/object and number/string pairs, but no RegExp/string pair. | A fixed source-text probe and generated RegExps compare with their own source text in both directions. | + +The `deepEquals` documentation names Date and RegExp as supported value +classes. The old examples supply the more precise rule that their primitive +encodings are not coerced into those classes. The receiving model assigns +expected inequality from those distinct value classes; it does not call the +production classifier. The fixed operands preserve every old assertion, and +the generated properties extend each missing boundary across finite Dates and +valid RegExps. A deliberately coercing Date/number, Date/string, or +RegExp/string comparator fails the receiving checker at its forward inequality +assertion. Both directions are checked for the real production function. + +The two legacy blocks contained six tests. After transfer, the focused oracle +and remaining unit file passed 208 tests with no type errors. The generator +uses finite Date timestamps from -1,000,000,000,000 through 2,000,000,000,000 +milliseconds and the existing valid RegExp source/flag grammar; fixed and +seedless campaigns use the same properties. This establishes the direct +`deepEquals` return for those pairs. It does not establish cross-realm values, +RegExp subclasses, arbitrary prototype changes, every cross-type pair, or a +public change-event trace. Those need their own owner or receiving witness. + +## Oracle guide audit + +| Requirement | Evidence and limit | +| --- | --- | +| ORC-001 | Existing `deepEquals` documentation and retained examples supply the class-preserving law; the direct value classes and return checkpoint are explicit. | +| ORC-002 | Separate constructors and primitive encodings determine expected copy, change, and cross-type outcomes without production classification. | +| ORC-003 | The receiver explains its law, bounded grammar, production call, and return comparison beside the checks. | +| ORC-004 | Every removed operand and outcome has a fixed reconstruction; generated finite Dates and RegExps vary the new cross-type boundaries. | +| ORC-005 | Each check calls exported `deepEquals` and observes the immediate boolean return. No public publication claim is made. | +| ORC-006 | Injected primitive-coercing comparators fail the exact inequality assertion for all three newly recovered boundaries. | +| ORC-007 | `utilsProperty` runs the same property with fixed and random seeds. The source/flags property replayed a captured failing seed/path directly, as recorded below. | +| ORC-008 | Not triggered: the value model has no mutable state. | +| ORC-009 | Date, RegExp, ISO text, epoch number, and source text are distinct value classes in this model; no production storage state is modeled. | +| ORC-010 | The direct check owns no cleanup resource; its assertion is the failure checkpoint. | +| ORC-011 | The source-isolated audits found a shared broad-category omission. The old exact examples and generated checks now enforce each primitive boundary; no independent second production formulation is claimed. | +| ORC-012 | This ledger records the source assertions, recovered losses, receiving checks, execution, limits, and pending exact-commit review. | +| ORC-013 | A primitive with matching displayed or numeric content distinguishes class-preserving equality from a plausible coercing design. | +| ORC-014 | Not triggered: the direct call has no provider or host boundary. | + +## Exact-commit hostile review and follow-up + +At committed head `c4c07da42`, the fresh reviewer confirmed that all six +retired assertions had fixed receiving witnesses and found no production bug +in this transfer. It found two broader oracle weaknesses: the generated +RegExp law varied flags but not source within a pair, and the two newly +registered cross-type properties had shifted existing ordinal replay IDs. +The generated source law now compares a same-flags, changed-source neighbor. +The two added properties are registered after the original properties, so +their old selectors retain their meanings. A focused run passed 139 oracle +tests with no type errors. + +For ORC-007 calibration, a temporary wrong model rule expected an arbitrary +RegExp to equal its changed-source neighbor. The fixed campaign failed at the +`deepEquals` return comparison after one test with seed `20260911` and shrink +path `0:0`. Direct replay through `UTILS_ORACLE_PROPERTY=34`, +`UTILS_ORACLE_SEED=20260911`, and `UTILS_ORACLE_PATH=0:0` reproduced that +same assertion failure. Restoring the model rule made the direct replay pass. +The temporary rule is absent from the final oracle. This demonstrates replay +fidelity for the changed-source property, not every property in the file. + +At `ae0b67fd2`, a fresh hostile reviewer confirmed all six retired assertions +and found no production mismatch in this transfer. It caught a replay defect: +the two new cross-type registrations preceded the existing RegExp, Float32Array, +and Duration registrations, shifting saved selectors 34–36. Moving the new +registrations after them restored 34 to RegExp source and flags, 35 to +Float32Array, and 36 to Duration; the new properties are 37 and 38. A temporary +wrong expected result for the RegExp changed-source neighbor failed on direct +replay of selector 34, seed `20260911`, path `0:0`, after one test at the +forward `deepEquals` return. Removing the temporary rule made that replay pass +with only the selected generated property active. Fresh review of the next +committed head remains pending. + +At committed head `be5b7a3c4`, a fresh hostile reviewer found no blocker in +the six-test Date and RegExp transfer. All six retired assertions still have +fixed witnesses and broader generated laws. The owner passed 139 tests with +no type errors. Direct replays confirmed that selectors 34–38 name the +recorded properties, including selector 34 with seed `20260911` and path +`0:0`. The result covers direct `deepEquals` calls. It does not establish +public change events, cross-realm values, RegExp subclasses, or arbitrary +cross-type pairs. Ordinal selectors can shift if future registrations are +inserted ahead of them. + +## Frozen-head ledger recheck + +At `210651d207eefe9bbc025c780f6259e470225c9e`, the hostile reviewer +read the six retired Date/RegExp assertions, their source, the direct +`deepEquals` implementation, and the receiving oracle. Source inspection +found no missing operand or expected result in the bounded transfer. This +is a frozen-head no-loss verdict by inspection; it is not a new mutant kill +or a fresh committed-head execution of subsequent working-tree changes. + +## Exact code-commit review of the narrowed transfer + +At `24a61a8d5b6cef06d488c490e39f351b4d55fb68`, a fresh hostile reviewer +checked the committed six retired Date/RegExp assertions against their +fixed witnesses, generated laws, and production `deepEquals` path. It found +no loss at the direct-return checkpoint. The committed oracle and remaining +unit file passed 208 tests with no type errors. The scoped files and commit +check were clean. This verdict does not establish public change events or +cross-realm behavior. diff --git a/docs/reference/functions/compareTemporalValues.md b/docs/reference/functions/compareTemporalValues.md index 2f412b437a..a96de8a7ef 100644 --- a/docs/reference/functions/compareTemporalValues.md +++ b/docs/reference/functions/compareTemporalValues.md @@ -7,15 +7,12 @@ title: compareTemporalValues function compareTemporalValues(a, b): number; ``` -Defined in: [packages/db/src/utils/comparison.ts:313](https://github.com/TanStack/db/blob/main/packages/db/src/utils/comparison.ts#L313) +Defined in: [packages/db/src/utils/comparison.ts:300](https://github.com/TanStack/db/blob/main/packages/db/src/utils/comparison.ts#L300) Compare two Temporal values of the same type, returning -1, 0, or 1. -Dispatch is keyed on `Symbol.toStringTag` (the brand already checked by -`isTemporal`) rather than `a.constructor`, making it robust across realms -and resistant to a shadowed `constructor` property. Types without a static -`.compare` (e.g. `PlainMonthDay`) throw rather than fall back to string -comparison, matching Temporal's design intent. +The tags check type agreement. Types without a static `.compare` (e.g. +`PlainMonthDay`) throw instead of using text order. Callers must ensure both arguments are Temporal objects; mixed types throw. diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index a786f671cd..0673dd2f48 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -297,7 +297,6 @@ "loadPrefix": "e$", "activateLease": "e0", "calculateSize": "e1", - "compareStored": "e2", "invokeCleanup": "e3", "onceCallbacks": "e4", "publishEvents": "e5", @@ -380,5 +379,29 @@ "readyFailureSink": "gn", "writeExplicitRowMetadata": "a8", "applyOwnedMutations": "bX", - "commitDirect": "cn" + "commitDirect": "cn", + "defersAcquisition": "c_", + "_getCurrentBindings": "dH", + "subscriberOrPreloadListeners": "ex", + "changeAcquiringSubscribers": "e2", + "discardPreloadedLiveQuery": "fU", + "pendingInitialLoads": "f1", + "subscriberOrPreload": "f2", + "readsBoundedPrefix": "gp", + "captureMutations": "gq", + "startSyncIfStopped": "gr", + "trackInitialLoad": "gs", + "nullsFirst": "gu", + "sourcesPermitInitialReadiness": "gv", + "startLiveQueryPreload": "gw", + "getSourceInfo": "gx", + "publishReplayRows": "gy", + "afterPreloadMark": "gz", + "resumeSubscriber": "gA", + "beginWrite": "gB", + "detach": "gC", + "copyRows": "gD", + "restore": "gE", + "attach": "gF", + "sortedExactValues": "gG" } diff --git a/packages/db/package.json b/packages/db/package.json index 81dc7f5217..92d04b8bb1 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -23,7 +23,7 @@ "test": "vitest --run", "test:dist": "vitest run --config vitest.dist.config.ts", "test:facade-retention": "node --expose-gc --import tsx tests/facade-retention.probe.ts", - "test:oracles": "vitest --run --coverage.enabled=false tests/reverse-index-oracle.property.test.ts tests/query/compiler/in-evaluator-oracle.property.test.ts tests/cleanup-queue-oracle.property.test.ts tests/comparison-oracle.property.test.ts tests/cursor-oracle.property.test.ts tests/index-update-oracle.property.test.ts tests/utils-oracle.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/local-storage-order-oracle.test.ts tests/local-storage-peer-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/transaction-ownership-oracle.property.test.ts tests/optimistic-settlement-boundaries-oracle.test.ts tests/optimistic-history-publication-oracle.test.ts tests/optimistic-history-outcomes-oracle.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-index-value-oracle.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history-oracle.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history-oracle.property.test.ts tests/live-query-observer-snapshot-oracle.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-alias-shadowing-oracle.test.ts tests/query/includes-outer-join-route-work-oracle.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state-oracle.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/live-query-deferred-acquisition-oracle.test.ts tests/flat-change-tracking-oracle.property.test.ts tests/truncate-readiness-oracle.test.ts tests/proxy-detachment-contract-oracle.test.ts tests/production-error-messages-oracle.test.ts tests/query/bucket-facade-rollback-oracle.property.test.ts tests/query/aggregate-select-oracle.property.test.ts", + "test:oracles": "vitest --run --coverage.enabled=false tests/reverse-index-oracle.property.test.ts tests/query/compiler/in-evaluator-oracle.property.test.ts tests/cleanup-queue-oracle.property.test.ts tests/comparison-oracle.property.test.ts tests/duration-query-equality-oracle.test.ts tests/temporal-index-order-oracle.test.ts tests/cursor-oracle.property.test.ts tests/index-update-oracle.property.test.ts tests/utils-oracle.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/local-storage-order-oracle.test.ts tests/local-storage-peer-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/transaction-ownership-oracle.property.test.ts tests/optimistic-settlement-boundaries-oracle.test.ts tests/optimistic-history-publication-oracle.test.ts tests/optimistic-history-outcomes-oracle.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-index-value-oracle.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history-oracle.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history-oracle.property.test.ts tests/live-query-observer-snapshot-oracle.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/custom-index-order-selection-oracle.test.ts tests/query/mixed-index-order-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-alias-shadowing-oracle.test.ts tests/query/includes-outer-join-route-work-oracle.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state-oracle.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/live-query-deferred-acquisition-oracle.test.ts tests/flat-change-tracking-oracle.property.test.ts tests/truncate-readiness-oracle.test.ts tests/proxy-detachment-contract-oracle.test.ts tests/production-error-messages-oracle.test.ts tests/query/bucket-facade-rollback-oracle.property.test.ts tests/query/aggregate-select-oracle.property.test.ts", "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run", "test:indexed-db:package": "pnpm --filter @tanstack/db... build && vitest run --config vitest.indexed-db.package.config.ts", "test:indexed-db:browser": "playwright test --config playwright.indexed-db.config.ts", diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index f69ab13d43..4ad8e8aa56 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -356,7 +356,12 @@ function getOrderedKeys( // Find the index const index = findIndexForField(collection, fieldPath, compareOpts) - if (index && index.supports(`gt`)) { + if ( + index && + index.supports(`gt`) && + index.supportsRangeOptimization && + index.supportsOrderOptimization + ) { // Use index optimization const filterFn = (key: TKey): boolean => { const value = collection.get(key) diff --git a/packages/db/src/indexes/auto-index.ts b/packages/db/src/indexes/auto-index.ts index c72c82807a..0f1c9bb6f2 100644 --- a/packages/db/src/indexes/auto-index.ts +++ b/packages/db/src/indexes/auto-index.ts @@ -1,4 +1,5 @@ import { DEFAULT_COMPARE_OPTIONS } from '../utils' +import { indexableOperations } from '../query/builder/functions.js' import { hasVirtualPropPath } from '../virtual-props' import { codedWarning, devBuild } from '../error-message.js' import { checkCollectionSizeForIndex, isDevModeEnabled } from './index-registry' @@ -21,7 +22,6 @@ export function ensureIndexForField< fieldPath: Array, collection: CollectionImpl, compareOptions?: CompareOptions, - compareFn?: (a: any, b: any) => number, ) { if (hasVirtualPropPath(fieldPath)) { return @@ -69,7 +69,7 @@ export function ensureIndexForField< }, { name: `auto:${fieldPath.join(`.`)}`, - options: { compareFn, compareOptions: compareOpts }, + options: { compareOptions: compareOpts }, }, ) } catch (error) { @@ -123,8 +123,7 @@ function extractIndexableExpressions( } // Check if this is a supported operation - const supportedOperations = [`eq`, `gt`, `gte`, `lt`, `lte`, `in`] - if (!supportedOperations.includes(func.name)) { + if (!indexableOperations.includes(func.name)) { return } diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index b8f3a59bb4..4c1c80f6b5 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -1,7 +1,15 @@ import { compileSingleRowExpression } from '../query/compiler/evaluators.js' -import { comparisonFunctions } from '../query/builder/functions.js' +import { + comparisonFunctions, + indexableOperations, +} from '../query/builder/functions.js' import { DEFAULT_COMPARE_OPTIONS, deepEquals } from '../utils.js' -import { makeCheckedComparator, makeComparator } from '../utils/comparison.js' +import { + indexObjectKind, + makeCheckedComparator, + makeComparator, + normalizeValue, +} from '../utils/comparison.js' import { codedMessage, codedWarning, devBuild } from '../error-message.js' import type { CompiledSingleRowExpression } from '../query/compiler/evaluators.js' import type { RangeQueryOptions } from './btree-index.js' @@ -35,6 +43,8 @@ function resolveStringSort( */ export const IndexOperation = comparisonFunctions +export { indexableOperations } from '../query/builder/functions.js' + export const builtInIndexResolverNames = new WeakMap() /** @@ -42,6 +52,11 @@ export const builtInIndexResolverNames = new WeakMap() */ export type IndexOperation = (typeof comparisonFunctions)[number] +export interface IndexComparisonOptions { + compareFn?: (a: any, b: any) => number + compareOptions?: CompareOptions +} + /** The read-side surface consumers use on a resolved (possibly reversed) index. */ export type IndexReader = Pick< IndexInterface, @@ -52,6 +67,7 @@ export type IndexReader = Pick< | `keyCount` | `supports` | `supportsRangeOptimization` + | `supportsOrderOptimization` | `canOptimizeRangeFor` > @@ -100,6 +116,8 @@ export interface IndexInterface< * a full scan when this is `false`. */ get supportsRangeOptimization(): boolean + /** Whether a query can use this index for its declared ordered prefix. */ + readonly supportsOrderOptimization?: boolean /** * Whether the live values in this index share the predicate operand's @@ -122,7 +140,7 @@ export abstract class BaseIndex< public readonly id: number public readonly name?: string public readonly expression: BasicExpression - public abstract readonly supportedOperations: Set + public readonly supportedOperations: Set protected readonly compareOptions: CompareOptions /** * Orders indexed values. Every result is checked, because a custom @@ -148,9 +166,12 @@ export abstract class BaseIndex< this.compareOptions = options?.compareOptions ?? DEFAULT_COMPARE_OPTIONS this.hasCustomComparator = options?.compareFn != null this.compareFn = makeCheckedComparator( - options?.compareFn ?? makeComparator(this.compareOptions), + options?.compareFn + ? (a, b) => options.compareFn(normalizeValue(a), normalizeValue(b)) + : makeComparator(this.compareOptions, `index`), ) this.name = name + this.supportedOperations = new Set(indexableOperations) this.initialize(options) } @@ -181,10 +202,38 @@ export abstract class BaseIndex< ): Array abstract get keyCount(): number abstract equalityLookup(value: any): Set - abstract inArrayLookup(values: Array): Set + protected exactBucketKeys(value: any): Iterable | undefined { + return this.equalityLookup(value) + } abstract rangeQuery(options: RangeQueryOptions): Set // Common methods + protected lookupSupportedOperation( + operation: IndexOperation, + value: any, + ): Set | undefined { + if (operation === `eq`) return this.equalityLookup(value) + if (operation === `in`) return this.inArrayLookup(value) + if (operation === `gt` || operation === `gte`) + return this.rangeQuery({ + from: value, + fromInclusive: operation === `gte`, + }) + if (operation === `lt` || operation === `lte`) + return this.rangeQuery({ to: value, toInclusive: operation === `lte` }) + return undefined + } + + /** Combine exact buckets without copying each bucket first. */ + inArrayLookup(values: Array): Set { + const result = new Set() + for (const value of values) { + const keys = this.exactBucketKeys(value) + if (keys) for (const key of keys) result.add(key) + } + return result + } + rangeQueryReversed(options: RangeQueryOptions = {}): Set { const { from, to, fromInclusive = true, toInclusive = true } = options const reversed: RangeQueryOptions = {} @@ -207,6 +256,24 @@ export abstract class BaseIndex< return !this.hasCustomComparator } + get supportsOrderOptimization(): boolean { + if (this.hasCustomComparator || this.rangeValueDomains.size > 1) + return false + // Within these domains the index orders by a storage key (or lacks a + // defined query order), rather than by the query's value comparator. + const domain = this.rangeValueDomains.keys().next().value + return ( + domain === undefined || + ![ + `Array`, + `Binary`, + `Temporal.Duration.calendar`, + `Temporal.PlainMonthDay`, + `function`, + ].includes(domain) + ) + } + protected addRangeValue(value: unknown): void { const domain = rangeValueDomain(value) if (domain === undefined) return @@ -295,7 +362,7 @@ export abstract class BaseIndex< return this.compareOptions.direction === direction } - protected abstract initialize(options?: any): void + protected initialize(_options?: any): void {} protected evaluateIndexExpression(item: any): any { const evaluator = (this.compiledIndexEvaluator ??= @@ -321,6 +388,7 @@ export abstract class BaseIndex< function rangeValueDomain(value: unknown): string | undefined { if (value == null) return undefined if (value instanceof Date) return `date` + if (typeof value === `object`) return indexObjectKind(value) return typeof value } diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index 0e5c91eec6..911959171d 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -1,34 +1,27 @@ import { compareKeys } from '@tanstack/db-ivm' import { areSameValueZeroEqual, normalizeValue } from '../utils/comparison.js' +import { codedMessage, devBuild } from '../error-message.js' import { compareKeysReversed, findInsertPositionInArray, } from '../utils/array-utils.js' -import { codedMessage, devBuild } from '../error-message.js' import { BaseIndex, builtInIndexResolverNames, warnRemovalFailure, } from './base-index.js' -import type { CompareOptions } from '../query/builder/types.js' -import type { IndexOperation } from './base-index.js' +import type { IndexComparisonOptions, IndexOperation } from './base-index.js' +import type { RangeQueryOptions } from './btree-index.js' -/** - * Options for range queries - */ -export interface RangeQueryOptions { - from?: any - to?: any - fromInclusive?: boolean - toInclusive?: boolean -} +export type { RangeQueryOptions } from './btree-index.js' /** * Options for Basic index */ -export interface BasicIndexOptions { - compareFn?: (a: any, b: any) => number - compareOptions?: CompareOptions +export type BasicIndexOptions = IndexComparisonOptions + +function orderValue(value: unknown): unknown { + return value instanceof Date ? value.getTime() : value } /** @@ -44,24 +37,14 @@ export interface BasicIndexOptions { export class BasicIndex< TKey extends string | number = string | number, > extends BaseIndex { - public readonly supportedOperations = new Set([ - `eq`, - `gt`, - `gte`, - `lt`, - `lte`, - `in`, - ]) - // Map for O(1) equality lookups: indexedValue -> Set of PKs private valueMap = new Map>() // Sorted array of unique indexed values for range queries private sortedValues: Array = [] + private sortedExactValues: Array = [] // Set of all indexed PKs private indexedKeys = new Set() - protected initialize(_options?: BasicIndexOptions): void {} - /** * Adds a value to the index */ @@ -69,13 +52,17 @@ export class BasicIndex< const indexedValue = this.evaluateAddedValue(key, item) const normalizedValue = normalizeValue(indexedValue) - this.addToBucket(key, normalizedValue) + this.addToBucket(key, normalizedValue, indexedValue) this.addRangeValue(indexedValue) this.indexedKeys.add(key) } - private addToBucket(key: TKey, normalizedValue: unknown): void { + private addToBucket( + key: TKey, + normalizedValue: unknown, + value: unknown, + ): void { const keySet = this.valueMap.get(normalizedValue) if (keySet) { // Value already exists, just add the key to the set @@ -83,13 +70,15 @@ export class BasicIndex< } else { // New value. Find its sorted position first: a comparator failure must // throw before the index changes. + const orderedValue = orderValue(value) const insertIdx = findInsertPositionInArray( this.sortedValues, - normalizedValue, + orderedValue, this.compareFn, ) this.valueMap.set(normalizedValue, new Set([key])) - this.sortedValues.splice(insertIdx, 0, normalizedValue) + this.sortedValues.splice(insertIdx, 0, orderedValue) + this.sortedExactValues.splice(insertIdx, 0, normalizedValue) } } @@ -108,35 +97,41 @@ export class BasicIndex< const normalizedValue = normalizeValue(indexedValue) - this.removeFromBucket(key, normalizedValue) + this.removeFromBucket(key, normalizedValue, indexedValue) this.removeRangeValue(indexedValue) this.indexedKeys.delete(key) } - private removeFromBucket(key: TKey, normalizedValue: unknown): void { + private removeFromBucket( + key: TKey, + normalizedValue: unknown, + value: unknown, + ): void { const keySet = this.valueMap.get(normalizedValue) if (keySet?.has(key)) { // The last key for this value leaves the sorted array too. Locate it // before any change, so a comparator failure leaves the index unchanged. if (keySet.size === 1) { + const orderedValue = orderValue(value) let sortedIndex = findInsertPositionInArray( this.sortedValues, - normalizedValue, + orderedValue, this.compareFn, ) // Distinct equality keys may share one comparator position. while ( sortedIndex < this.sortedValues.length && - this.compareFn(this.sortedValues[sortedIndex], normalizedValue) === 0 + this.compareFn(this.sortedValues[sortedIndex], orderedValue) === 0 ) { if ( areSameValueZeroEqual( - this.sortedValues[sortedIndex], + this.sortedExactValues[sortedIndex], normalizedValue, ) ) { this.sortedValues.splice(sortedIndex, 1) + this.sortedExactValues.splice(sortedIndex, 1) break } sortedIndex++ @@ -174,9 +169,9 @@ export class BasicIndex< return } - this.removeFromBucket(key, oldValue) + this.removeFromBucket(key, oldValue, oldIndexedValue) this.removeRangeValue(oldIndexedValue) - this.addToBucket(key, newValue) + this.addToBucket(key, newValue, newIndexedValue) this.addRangeValue(newIndexedValue) this.indexedKeys.add(key) } @@ -187,26 +182,29 @@ export class BasicIndex< build(entries: Iterable<[TKey, any]>): void { this.clear() - // Collect all entries first - const entriesArray: Array<{ key: TKey; value: any }> = [] for (const [key, item] of entries) { const indexedValue = this.evaluateAddedValue(key, item) - entriesArray.push({ key, value: normalizeValue(indexedValue) }) - this.addRangeValue(indexedValue) - this.indexedKeys.add(key) - } - - // Group by value - for (const { key, value } of entriesArray) { - if (this.valueMap.has(value)) { - this.valueMap.get(value)!.add(key) - } else { + const value = normalizeValue(indexedValue) + const keys = this.valueMap.get(value) + if (keys) keys.add(key) + else { this.valueMap.set(value, new Set([key])) + this.sortedValues.push(orderValue(indexedValue)) + this.sortedExactValues.push(value) } + this.addRangeValue(indexedValue) + this.indexedKeys.add(key) } - // Build sorted array from unique values - this.sortedValues = Array.from(this.valueMap.keys()).sort(this.compareFn) + // Array.sort moves raw undefined values last without comparing them. + // Wrap values so nulls-first comparison runs during a full build. + const sortable = this.sortedValues.map((value, index) => ({ + value, + exact: this.sortedExactValues[index], + })) + sortable.sort((left, right) => this.compareFn(left.value, right.value)) + this.sortedValues = sortable.map(({ value }) => value) + this.sortedExactValues = sortable.map(({ exact }) => exact) } /** @@ -215,43 +213,19 @@ export class BasicIndex< clear(): void { this.valueMap.clear() this.sortedValues = [] + this.sortedExactValues = [] this.indexedKeys.clear() this.clearRangeValues() } - /** - * Performs a lookup operation - */ lookup(operation: IndexOperation, value: any): Set { - let result: Set - - switch (operation) { - case `eq`: - result = this.equalityLookup(value) - break - case `gt`: - result = this.rangeQuery({ from: value, fromInclusive: false }) - break - case `gte`: - result = this.rangeQuery({ from: value, fromInclusive: true }) - break - case `lt`: - result = this.rangeQuery({ to: value, toInclusive: false }) - break - case `lte`: - result = this.rangeQuery({ to: value, toInclusive: true }) - break - case `in`: - result = this.inArrayLookup(value) - break - default: - throw new Error( - devBuild() && process.env.NODE_ENV !== `production` - ? `Operation ${operation} not supported by BasicIndex` - : codedMessage(100, { operation }), - ) - } - return result + const result = this.lookupSupportedOperation(operation, value) + if (result !== undefined) return result + throw new Error( + devBuild() && process.env.NODE_ENV !== `production` + ? `Operation ${operation} not supported by BasicIndex` + : codedMessage(100, { operation }), + ) } /** @@ -280,8 +254,8 @@ export class BasicIndex< return result } - const normalizedFrom = normalizeValue(from) - const normalizedTo = normalizeValue(to) + const orderedFrom = orderValue(from) + const orderedTo = orderValue(to) const hasFrom = `from` in options const hasTo = `to` in options @@ -290,7 +264,7 @@ export class BasicIndex< if (hasFrom) { startIdx = findInsertPositionInArray( this.sortedValues, - normalizedFrom, + orderedFrom, this.compareFn, ) // Comparator-equal values form one range boundary even when they are @@ -298,7 +272,7 @@ export class BasicIndex< while ( !fromInclusive && startIdx < this.sortedValues.length && - this.compareFn(this.sortedValues[startIdx], normalizedFrom) === 0 + this.compareFn(this.sortedValues[startIdx], orderedFrom) === 0 ) { startIdx++ } @@ -309,14 +283,14 @@ export class BasicIndex< if (hasTo) { endIdx = findInsertPositionInArray( this.sortedValues, - normalizedTo, + orderedTo, this.compareFn, ) // Include the whole comparator group at an inclusive upper boundary. while ( toInclusive && endIdx < this.sortedValues.length && - this.compareFn(this.sortedValues[endIdx], normalizedTo) === 0 + this.compareFn(this.sortedValues[endIdx], orderedTo) === 0 ) { endIdx++ } @@ -324,7 +298,7 @@ export class BasicIndex< // Collect all keys in range for (let i = startIdx; i < endIdx; i++) { - const keys = this.valueMap.get(this.sortedValues[i]) + const keys = this.valueMap.get(this.sortedExactValues[i]) if (keys) { keys.forEach((key) => result.add(key)) } @@ -337,16 +311,16 @@ export class BasicIndex< * Returns the next n items in sorted order */ take(n: number, from: any, filterFn?: (key: TKey) => boolean): Array { - const normalizedFrom = normalizeValue(from) + const orderedFrom = orderValue(from) let startIdx = findInsertPositionInArray( this.sortedValues, - normalizedFrom, + orderedFrom, this.compareFn, ) // Skip past the 'from' value (exclusive) while ( startIdx < this.sortedValues.length && - this.compareFn(this.sortedValues[startIdx], normalizedFrom) <= 0 + this.compareFn(this.sortedValues[startIdx], orderedFrom) <= 0 ) { startIdx++ } @@ -362,17 +336,17 @@ export class BasicIndex< from: any, filterFn?: (key: TKey) => boolean, ): Array { - const normalizedFrom = normalizeValue(from) + const orderedFrom = orderValue(from) let startIdx = findInsertPositionInArray( this.sortedValues, - normalizedFrom, + orderedFrom, this.compareFn, ) - 1 // Skip past the 'from' value (exclusive) while ( startIdx >= 0 && - this.compareFn(this.sortedValues[startIdx], normalizedFrom) >= 0 + this.compareFn(this.sortedValues[startIdx], orderedFrom) >= 0 ) { startIdx-- } @@ -413,7 +387,8 @@ export class BasicIndex< const groupValue = this.sortedValues[index] const groupKeys: Array = [] do { - for (const key of this.valueMap.get(this.sortedValues[index]) ?? []) { + for (const key of this.valueMap.get(this.sortedExactValues[index]) ?? + []) { groupKeys.push(key) } index += step @@ -430,23 +405,6 @@ export class BasicIndex< } return result } - - /** - * Performs an IN array lookup - O(k) where k is values.length - */ - inArrayLookup(values: Array): Set { - const result = new Set() - - for (const value of values) { - const normalizedValue = normalizeValue(value) - const keys = this.valueMap.get(normalizedValue) - if (keys) { - keys.forEach((key) => result.add(key)) - } - } - - return result - } } builtInIndexResolverNames.set(BasicIndex, `BasicIndex`) diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 2266c18a1f..84d5a7acae 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -1,31 +1,16 @@ import { compareKeys } from '@tanstack/db-ivm' import { compareKeysReversed } from '../utils/array-utils.js' import { BTree } from '../utils/btree.js' -import { - areSameValueZeroEqual, - denormalizeUndefined, - normalizeForBTree, -} from '../utils/comparison.js' +import { areSameValueZeroEqual, normalizeValue } from '../utils/comparison.js' import { codedMessage, devBuild } from '../error-message.js' import { BaseIndex, builtInIndexResolverNames, warnRemovalFailure, } from './base-index.js' -import type { CompareOptions } from '../query/builder/types.js' -import type { IndexOperation } from './base-index.js' - -/** - * Options for Ordered index - */ -export interface BTreeIndexOptions { - compareFn?: (a: any, b: any) => number - compareOptions?: CompareOptions -} +import type { IndexComparisonOptions, IndexOperation } from './base-index.js' -/** - * Options for range queries - */ +export type BTreeIndexOptions = IndexComparisonOptions export interface RangeQueryOptions { from?: any to?: any @@ -33,186 +18,130 @@ export interface RangeQueryOptions { toInclusive?: boolean } -type OrderedBucket = { - representative: unknown - exactValues: Set - keys: Set +type SortKey = { value: any } +type Group = SortKey & { + keys: Map + buckets?: Map> + representativeKey: TKey + nextRepresentative?: IterableIterator<[TKey, any]> } - -/** - * B+Tree index for sorted data with range queries - * This maintains items in sorted order and provides efficient range operations - */ export class BTreeIndex< TKey extends string | number = string | number, > extends BaseIndex { - public readonly supportedOperations = new Set([ - `eq`, - `gt`, - `gte`, - `lt`, - `lte`, - `in`, - ]) - - // Internal data structures - private to hide implementation details - // The `orderedEntries` B+ tree groups values that occupy the same comparator - // position. The `valueMap` keeps exact values separate for equality lookups. - private orderedEntries = new BTree>((a, b) => - this.compareStored(a, b), + private orderedEntries = new BTree>((a, b) => + this.compareFn(a.value, b.value), ) - private valueMap = new Map< - unknown, - { keys: Set; ordered: OrderedBucket } - >() + private valueMap = new Map>() private indexedKeys = new Set() - // Stored keys use UNDEFINED_SENTINEL; the comparator, which may be - // user-supplied, expects undefined. - private compareStored(a: any, b: any): number { - return this.compareFn(denormalizeUndefined(a), denormalizeUndefined(b)) - } - - protected initialize(_options?: BTreeIndexOptions): void {} - - /** - * Adds a value to the index - */ add(key: TKey, item: any): void { - const indexedValue = this.evaluateAddedValue(key, item) - - // Normalize the value for Map key usage - const normalizedValue = normalizeForBTree(indexedValue) - - this.addToBucket(key, normalizedValue) - this.addRangeValue(indexedValue) - + const value = this.evaluateAddedValue(key, item) + const exact = normalizeValue(value) + this.addToBucket(key, value, exact) + this.addRangeValue(value) this.indexedKeys.add(key) } - private addToBucket(key: TKey, normalizedValue: unknown): void { - const exact = this.valueMap.get(normalizedValue) - if (exact) { - exact.keys.add(key) - exact.ordered.keys.add(key) + private addToBucket(key: TKey, value: any, exact: unknown): void { + const group = this.valueMap.get(exact) + if (group) { + group.keys.set(key, value) + group.buckets?.get(exact)!.add(key) return } - - let orderedBucket = this.orderedEntries.get(normalizedValue) - if (orderedBucket) { - orderedBucket.keys.add(key) - orderedBucket.exactValues.add(normalizedValue) + const ordered = this.orderedEntries.get({ value }) + if (ordered) { + ordered.buckets ??= new Map([ + [normalizeValue(ordered.value), new Set(ordered.keys.keys())], + ]) + ordered.buckets.set(exact, new Set([key])) + ordered.keys.set(key, value) + this.valueMap.set(exact, ordered) } else { - orderedBucket = { - representative: normalizedValue, - exactValues: new Set([normalizedValue]), - keys: new Set([key]), + const created = { + value, + keys: new Map([[key, value]]), + representativeKey: key, } - this.orderedEntries.set(normalizedValue, orderedBucket) + // The tree compares before the exact lookup map changes. + this.orderedEntries.set(created, created) + this.valueMap.set(exact, created) } - this.valueMap.set(normalizedValue, { - keys: new Set([key]), - ordered: orderedBucket, - }) } - /** - * Removes a value from the index - */ remove(key: TKey, item: any): void { - let indexedValue: any + let value: any try { - indexedValue = this.evaluateIndexExpression(item) + value = this.evaluateIndexExpression(item) } catch (error) { warnRemovalFailure(key, error) return } - - // Normalize the value for Map key usage - const normalizedValue = normalizeForBTree(indexedValue) - - this.removeFromBucket(key, normalizedValue) - this.removeRangeValue(indexedValue) - + const exact = normalizeValue(value) + this.removeFromBucket(key, exact) + this.removeRangeValue(value) this.indexedKeys.delete(key) } - private removeFromBucket(key: TKey, normalizedValue: unknown): void { - const exact = this.valueMap.get(normalizedValue) - if (!exact?.keys.has(key)) return - const orderedBucket = exact.ordered - const removedExactValue = exact.keys.size === 1 - const replacesRepresentative = - removedExactValue && - orderedBucket.keys.size > 1 && - areSameValueZeroEqual(orderedBucket.representative, normalizedValue) - - // Tree writes compare first, so a comparator failure throws before the - // index changes. A comparator-equal `set` overwrites the entry's key. - if (orderedBucket.keys.size === 1) { - this.orderedEntries.delete(normalizedValue) - } else if (replacesRepresentative) { - const representative = [...orderedBucket.exactValues].find( - (value) => !areSameValueZeroEqual(value, normalizedValue), - ) - this.orderedEntries.set(representative, orderedBucket) - orderedBucket.representative = representative + private removeFromBucket(key: TKey, exact: unknown): void { + const group = this.valueMap.get(exact) + if (!group?.keys.has(key)) return + const bucket = group.buckets?.get(exact) + if (group.buckets && !bucket?.has(key)) return + if (group.keys.size === 1) { + this.orderedEntries.delete(group) } - exact.keys.delete(key) - orderedBucket.keys.delete(key) - if (removedExactValue) { - this.valueMap.delete(normalizedValue) - orderedBucket.exactValues.delete(normalizedValue) + bucket?.delete(key) + group.keys.delete(key) + if (bucket?.size === 0 || group.keys.size === 0) { + this.valueMap.delete(exact) + group.buckets?.delete(exact) + } + if (group.keys.size && group.representativeKey === key) { + // Advancing one iterator skips each removed Map slot at most once. + const [nextKey, nextValue] = (group.nextRepresentative ??= + group.keys.entries()).next().value as [TKey, any] + group.representativeKey = nextKey + group.value = nextValue } } - /** - * Updates a value in the index - */ update(key: TKey, oldItem: any, newItem: any): void { - let oldIndexedValue: unknown - let newIndexedValue: unknown + let oldValue: any + let newValue: any try { - oldIndexedValue = this.evaluateIndexExpression(oldItem) - newIndexedValue = this.evaluateIndexExpression(newItem) + oldValue = this.evaluateIndexExpression(oldItem) + newValue = this.evaluateIndexExpression(newItem) } catch { this.remove(key, oldItem) this.add(key, newItem) return } - - const oldValue = normalizeForBTree(oldIndexedValue) - const newValue = normalizeForBTree(newIndexedValue) - if ( - areSameValueZeroEqual(oldValue, newValue) && - this.valueMap.get(newValue)?.keys.has(key) - ) { - this.removeRangeValue(oldIndexedValue) - this.addRangeValue(newIndexedValue) + const oldExact = normalizeValue(oldValue) + const newExact = normalizeValue(newValue) + const group = this.valueMap.get(newExact) + const containsKey = group?.buckets + ? group.buckets.get(newExact)?.has(key) + : group?.keys.has(key) + if (areSameValueZeroEqual(oldExact, newExact) && containsKey) { + group!.keys.set(key, newValue) + if (group!.representativeKey === key) group!.value = newValue + this.removeRangeValue(oldValue) + this.addRangeValue(newValue) return } - - this.removeFromBucket(key, oldValue) - this.removeRangeValue(oldIndexedValue) - this.addToBucket(key, newValue) - this.addRangeValue(newIndexedValue) + this.removeFromBucket(key, oldExact) + this.removeRangeValue(oldValue) + this.addToBucket(key, newValue, newExact) + this.addRangeValue(newValue) this.indexedKeys.add(key) } - /** - * Builds the index from a collection of entries - */ build(entries: Iterable<[TKey, any]>): void { this.clear() - - for (const [key, item] of entries) { - this.add(key, item) - } + for (const [key, item] of entries) this.add(key, item) } - /** - * Clears all data from the index - */ clear(): void { this.orderedEntries.clear() this.valueMap.clear() @@ -220,211 +149,101 @@ export class BTreeIndex< this.clearRangeValues() } - /** - * Performs a lookup operation - */ lookup(operation: IndexOperation, value: any): Set { - let result: Set - - switch (operation) { - case `eq`: - result = this.equalityLookup(value) - break - case `gt`: - result = this.rangeQuery({ from: value, fromInclusive: false }) - break - case `gte`: - result = this.rangeQuery({ from: value, fromInclusive: true }) - break - case `lt`: - result = this.rangeQuery({ to: value, toInclusive: false }) - break - case `lte`: - result = this.rangeQuery({ to: value, toInclusive: true }) - break - case `in`: - result = this.inArrayLookup(value) - break - default: - throw new Error( - devBuild() && process.env.NODE_ENV !== `production` - ? `Operation ${operation} not supported by BTreeIndex` - : codedMessage(101, { operation }), - ) - } - return result + const result = this.lookupSupportedOperation(operation, value) + if (result !== undefined) return result + throw new Error( + devBuild() && process.env.NODE_ENV !== `production` + ? `Operation ${operation} not supported by BTreeIndex` + : codedMessage(101, { operation }), + ) } - /** - * Gets the number of indexed keys - */ get keyCount(): number { return this.indexedKeys.size } - // Public methods for backward compatibility (used by tests) - - /** - * Performs an equality lookup - */ equalityLookup(value: any): Set { - const normalizedValue = normalizeForBTree(value) - return new Set(this.valueMap.get(normalizedValue)?.keys ?? []) + return new Set(this.exactBucketKeys(value)) + } + + protected exactBucketKeys(value: any): Iterable | undefined { + const exact = normalizeValue(value) + const group = this.valueMap.get(exact) + return group?.buckets ? group.buckets.get(exact) : group?.keys.keys() } - /** - * Performs a range query with options - * This is more efficient for compound queries like "WHERE a > 5 AND a < 10" - */ rangeQuery(options: RangeQueryOptions = {}): Set { - const { from, to, fromInclusive = true, toInclusive = true } = options const result = new Set() - - // Check if from/to were explicitly provided (even if undefined) - // vs not provided at all (should use min/max key) - const hasFrom = `from` in options - const hasTo = `to` in options - - const fromKey = hasFrom - ? normalizeForBTree(from) - : this.orderedEntries.minKey() - const toKey = hasTo ? normalizeForBTree(to) : this.orderedEntries.maxKey() - + if (this.orderedEntries.size === 0) return result + const { from, to, fromInclusive = true, toInclusive = true } = options + const low = + 'from' in options ? { value: from } : this.orderedEntries.minKey()! + const high = 'to' in options ? { value: to } : this.orderedEntries.maxKey()! this.orderedEntries.forRange( - fromKey, - toKey, - toInclusive, - (indexedValue, bucket) => { - // Only exclude the boundary when an exclusive lower bound was - // actually provided. Without a `from` bound, `fromKey` defaults to - // the minimum key and must not be dropped. Compare against the - // normalized key since indexed values are stored normalized - // (e.g. dates as timestamps), so the raw `from` would never match. + low, + high, + !('to' in options) || toInclusive, + (sortKey, group) => { if ( - hasFrom && + 'from' in options && !fromInclusive && - this.compareStored(indexedValue, fromKey) === 0 - ) { - // the B+ tree `forRange` method does not support exclusive lower bounds - // so we need to exclude it manually + this.compareFn(sortKey.value, from) === 0 + ) return - } - - bucket.keys.forEach((key) => result.add(key)) + for (const key of group.keys.keys()) result.add(key) }, ) - return result } - /** - * Internal method for taking items from the index. - * @param n - The number of items to return - * @param nextPair - Function to get the next pair from the BTree - * @param from - Already normalized! undefined means "start from beginning/end", sentinel means "start from the key undefined" - * @param filterFn - Optional filter function - * @param reversed - Whether to reverse the order of keys within each value - */ private takeInternal( n: number, - nextPair: (k?: any) => [any, OrderedBucket] | undefined, - from: any, - filterFn?: (key: TKey) => boolean, - reversed: boolean = false, + from: SortKey | undefined, + filterFn: ((key: TKey) => boolean) | undefined, + reversed: boolean, ): Array { const result: Array = [] - let pair: [any, OrderedBucket] | undefined - let key = from // Use as-is - it's already normalized by the caller - - // Every key owns exactly one bucket, so the walk never repeats a key. - while ((pair = nextPair(key)) !== undefined && result.length < n) { - key = pair[0] - // Sort keys for deterministic order within a comparator position. - const sorted = Array.from(pair[1].keys).sort( + let pair = reversed + ? this.orderedEntries.nextLowerPair(from) + : this.orderedEntries.nextHigherPair(from) + while (pair && result.length < n) { + for (const key of [...pair[1].keys.keys()].sort( reversed ? compareKeysReversed : compareKeys, - ) - for (const ks of sorted) { + )) { + if (filterFn?.(key) ?? true) result.push(key) if (result.length >= n) break - if (filterFn?.(ks) ?? true) result.push(ks) } + if (result.length >= n) break + pair = reversed + ? this.orderedEntries.nextLowerPair(pair[0]) + : this.orderedEntries.nextHigherPair(pair[0]) } - return result } - /** - * Returns the next n items after the provided item. - * @param n - The number of items to return - * @param from - The item to start from (exclusive). - * @returns The next n items after the provided key. - */ take(n: number, from: any, filterFn?: (key: TKey) => boolean): Array { - const nextPair = (k?: any) => this.orderedEntries.nextHigherPair(k) - // Normalize the from value - const normalizedFrom = normalizeForBTree(from) - return this.takeInternal(n, nextPair, normalizedFrom, filterFn) + return this.takeInternal(n, { value: from }, filterFn, false) } - /** - * Returns the first n items from the beginning. - * @param n - The number of items to return - * @param filterFn - Optional filter function - * @returns The first n items - */ takeFromStart(n: number, filterFn?: (key: TKey) => boolean): Array { - const nextPair = (k?: any) => this.orderedEntries.nextHigherPair(k) - // Pass undefined to mean "start from beginning" (BTree's native behavior) - return this.takeInternal(n, nextPair, undefined, filterFn) + return this.takeInternal(n, undefined, filterFn, false) } - /** - * Returns the next n items **before** the provided item (in descending order). - * @param n - The number of items to return - * @param from - The item to start from (exclusive). Required. - * @returns The next n items **before** the provided key. - */ takeReversed( n: number, from: any, filterFn?: (key: TKey) => boolean, ): Array { - const nextPair = (k?: any) => this.orderedEntries.nextLowerPair(k) - // Normalize the from value - const normalizedFrom = normalizeForBTree(from) - return this.takeInternal(n, nextPair, normalizedFrom, filterFn, true) + return this.takeInternal(n, { value: from }, filterFn, true) } - /** - * Returns the last n items from the end. - * @param n - The number of items to return - * @param filterFn - Optional filter function - * @returns The last n items - */ takeReversedFromEnd( n: number, filterFn?: (key: TKey) => boolean, ): Array { - const nextPair = (k?: any) => this.orderedEntries.nextLowerPair(k) - // Pass undefined to mean "start from end" (BTree's native behavior) - return this.takeInternal(n, nextPair, undefined, filterFn, true) - } - - /** - * Performs an IN array lookup - */ - inArrayLookup(values: Array): Set { - const result = new Set() - - for (const value of values) { - const normalizedValue = normalizeForBTree(value) - const keys = this.valueMap.get(normalizedValue)?.keys - if (keys) { - keys.forEach((key) => result.add(key)) - } - } - - return result + return this.takeInternal(n, undefined, filterFn, true) } } -builtInIndexResolverNames.set(BTreeIndex, `BTreeIndex`) +builtInIndexResolverNames.set(BTreeIndex, 'BTreeIndex') diff --git a/packages/db/src/indexes/reverse-index.ts b/packages/db/src/indexes/reverse-index.ts index 9e018bc07f..181b5bd475 100644 --- a/packages/db/src/indexes/reverse-index.ts +++ b/packages/db/src/indexes/reverse-index.ts @@ -103,6 +103,10 @@ export class ReverseIndex< return this.originalIndex.supportsRangeOptimization } + get supportsOrderOptimization(): boolean { + return this.originalIndex.supportsOrderOptimization === true + } + canOptimizeRangeFor(value: unknown): boolean { return this.originalIndex.canOptimizeRangeFor?.(value) ?? true } diff --git a/packages/db/src/query/builder/functions.ts b/packages/db/src/query/builder/functions.ts index c2095a5cb6..c25c0bf772 100644 --- a/packages/db/src/query/builder/functions.ts +++ b/packages/db/src/query/builder/functions.ts @@ -704,13 +704,17 @@ export function max(arg: ExpressionLike): Aggregate { /** * List of comparison function names that can be used with indexes */ -export const comparisonFunctions = [ +export const indexableOperations = [ `eq`, `gt`, `gte`, `lt`, `lte`, `in`, +] as const + +export const comparisonFunctions = [ + ...indexableOperations, `like`, `ilike`, ] as const diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 88c0777f6c..b7e297dba6 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -189,19 +189,11 @@ export function processOrderBy( const compareOpts = buildCompareOptions(firstClause, collection) if (fieldName) { - // Use a single-column comparator for the index, not the - // multi-column `compare` function. The multi-column comparator - // expects array values [col1, col2, ...] but the index stores - // individual field values. Passing `compare` here causes the - // BTree to treat all single values as equal (since number[0] - // === undefined for both sides of the comparison). - const firstColumnCompareFn = makeComparator(compareOpts) ensureIndexForField( fieldName, followRefResult.path, followRefCollection, compareOpts, - firstColumnCompareFn, ) } @@ -211,8 +203,8 @@ export function processOrderBy( compareOpts, ) - // Only use the index if it supports range queries - if (!index?.supports(`gt`)) { + // An ordered source prefix must agree with the query's declared order. + if (!index?.supports(`gt`) || !index.supportsOrderOptimization) { index = undefined } diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index e472470ed2..ec99c9311b 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -9,6 +9,20 @@ interface TypedArray { [index: number]: number } +export const durationFields = [ + `years`, + `months`, + `weeks`, + `days`, + `hours`, + `minutes`, + `seconds`, + `milliseconds`, + `microseconds`, + `nanoseconds`, +] as const +export type DurationFields = Record<(typeof durationFields)[number], number> + /** * Deep equality function that compares two values recursively * Handles primitives, objects, arrays, Date, RegExp, Map, Set, TypedArrays, and Temporal objects @@ -263,6 +277,16 @@ export function deepEqualsInternal( // If they're different Temporal types, they're not equal if (aTag !== bTag) return false + // Duration identity preserves its fields, even when toString normalizes + // different units to the same elapsed time. + if (aTag === `Temporal.Duration`) { + const durationA = a as unknown as DurationFields + const durationB = b as unknown as DurationFields + return durationFields.every( + (field) => durationA[field] === durationB[field], + ) + } + // Use Temporal's built-in equals method if available if (typeof a.equals === `function`) { return a.equals(b) diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index eb6ac39dc3..1a4d79bac0 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -1,24 +1,9 @@ -import { isTemporal } from '../utils' +import { durationFields, isTemporal } from '../utils' import { getRuntimeReferenceIdentity } from '../query/runtime-reference-identity' import { codedMessage, devBuild } from '../error-message' +import type { DurationFields } from '../utils' import type { CompareOptions } from '../query/builder/types' -// WeakMap to store stable IDs for objects -const objectIds = new WeakMap() -let nextObjectId = 1 - -/** - * Get or create a stable ID for an object - */ -function getObjectId(obj: object): number { - if (objectIds.has(obj)) { - return objectIds.get(obj)! - } - const id = nextObjectId++ - objectIds.set(obj, id) - return id -} - /** * Whether a value has no IEEE-754 natural order: `NaN`, or an invalid Date * (whose timestamp is `NaN`). The query engine follows PostgreSQL float @@ -107,10 +92,10 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { const bIsObject = typeof b === `object` if (aIsObject || bIsObject) { - // If both are objects, compare their stable IDs + // Symbols and objects share the runtime's reference-identity sequence. if (aIsObject && bIsObject) { - const aId = getObjectId(a) - const bId = getObjectId(b) + const aId = getRuntimeReferenceIdentity(a)[2] + const bId = getRuntimeReferenceIdentity(b)[2] return aId - bId } @@ -134,21 +119,60 @@ export const descComparator = ( b: unknown, opts: CompareOptions, ): number => { - return ascComparator(b, a, { - ...opts, - nulls: opts.nulls === `first` ? `last` : `first`, - }) + return a == null || b == null + ? ascComparator(a, b, opts) + : ascComparator(b, a, opts) +} + +export function indexObjectKind(value: any): string { + if (isUint8Array(value)) return `Binary` + if (isTemporal(value)) { + const tag = value[Symbol.toStringTag] + const fields = value as unknown as DurationFields + return tag === `Temporal.Duration` && + (fields.years || fields.months || fields.weeks) + ? `Temporal.Duration.calendar` + : tag + } + if (Array.isArray(value)) return `Array` + return `Object` } export function makeComparator( opts: CompareOptions, + scope: `values` | `index` = `values`, ): (a: any, b: any) => number { + const compare = opts.direction === `asc` ? ascComparator : descComparator + if (scope !== `index`) return (a, b) => compare(a, b, opts) + + const direction = opts.direction === `asc` ? 1 : -1 return (a, b) => { - if (opts.direction === `asc`) { - return ascComparator(a, b, opts) - } else { - return descComparator(a, b, opts) + // An index needs one stable domain for each native comparison law. In + // particular, Date order cannot be mixed with reference-object order. + if (a != null && b != null) { + if (a instanceof Date) a = a.getTime() + if (b instanceof Date) b = b.getTime() + const aType = typeof a + const bType = typeof b + if (aType !== bType) return (aType < bType ? -1 : 1) * direction + if (aType === `object`) { + const aKind = indexObjectKind(a) + const bKind = indexObjectKind(b) + if (aKind !== bKind) return (aKind < bKind ? -1 : 1) * direction + // Binary content and unordered Temporal values use their exact keys + // for stable storage order. + if ( + aKind === `Binary` || + aKind === `Temporal.Duration.calendar` || + aKind === `Temporal.PlainMonthDay` + ) { + const left = normalizeValue(a) + const right = normalizeValue(b) + return (left < right ? -1 : left > right ? 1 : 0) * direction + } + } } + return compare(a, b, opts) } } @@ -200,33 +224,39 @@ function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean { } const NORMALIZED_KEY_PREFIX = `\u0000tanstack-db:` - function normalizedKey(kind: string, value: string): string { return `${NORMALIZED_KEY_PREFIX}${kind}:${value}` } +const binaryKeys = new WeakMap() + function normalizeBinary(value: Uint8Array): string { - let bytes = `` - for (let index = 0; index < value.byteLength; index++) { - bytes += String.fromCharCode(value[index]!) + const cached = binaryKeys.get(value) + if (cached) return cached + let bytes: string + if (value.byteLength <= 8192) { + bytes = String.fromCharCode.apply(null, value as unknown as Array) + } else { + const chunks: Array = [] + for (let index = 0; index < value.byteLength; index += 8192) { + chunks.push( + String.fromCharCode.apply( + null, + value.subarray(index, index + 8192) as unknown as Array, + ), + ) + } + bytes = chunks.join(``) } - return normalizedKey(`binary`, bytes) + const key = normalizedKey(`binary`, bytes) + binaryKeys.set(value, key) + return key } -/** - * Sentinel value representing undefined in normalized form. - * This allows distinguishing between "start from beginning" (undefined parameter) - * and "start from the key undefined" (actual undefined value in the tree). - */ -export const UNDEFINED_SENTINEL = normalizedKey(`undefined`, ``) - /** * Normalize a value for comparison and Map key usage * Converts values that can't be directly compared or used as Map keys * into comparable primitive representations - * - * Note: This does NOT convert undefined to a sentinel. Use normalizeForBTree - * for BTree index operations that need to distinguish undefined values. */ export function normalizeValue(value: any): any { if (typeof value === `string`) { @@ -244,10 +274,15 @@ export function normalizeValue(value: any): any { } if (isTemporal(value)) { - return normalizedKey( - `temporal`, - `${value[Symbol.toStringTag]}:${value.toString()}`, - ) + const tag = value[Symbol.toStringTag] + if (tag === 'Temporal.Duration') { + const fields = value as unknown as DurationFields + return normalizedKey( + tag, + durationFields.map((field) => fields[field]).join(','), + ) + } + return normalizedKey(tag, value.toString()) } // Normalize Uint8Arrays/Buffers to a string representation for Map key usage @@ -259,19 +294,6 @@ export function normalizeValue(value: any): any { return value } -/** - * Normalize a value for BTree index usage. - * Extends normalizeValue to also convert undefined to a sentinel value. - * This is needed because the BTree does not properly support `undefined` as a key - * (it interprets undefined as "start from beginning" in nextHigherPair/nextLowerPair). - */ -export function normalizeForBTree(value: any): any { - if (value === undefined) { - return UNDEFINED_SENTINEL - } - return normalizeValue(value) -} - /** * Compare values using the equality semantics used by Map keys. */ @@ -279,40 +301,17 @@ export function areSameValueZeroEqual(a: unknown, b: unknown): boolean { return a === b || (Number.isNaN(a) && Number.isNaN(b)) } -/** - * Converts the `UNDEFINED_SENTINEL` back to `undefined`. - * Needed such that the sentinel is converted back to `undefined` before comparison. - */ -export function denormalizeUndefined(value: any): any { - if (value === UNDEFINED_SENTINEL) { - return undefined - } - return value -} - -// Cached map from Symbol.toStringTag → static compare function (null = none defined). -// Populated lazily on first encounter of each Temporal type so we never access -// `.constructor` more than once per type, and dispatch is keyed on the already- -// computed brand tag rather than on the constructor itself. -const temporalCompareByTag = new Map< - string, - ((a: unknown, b: unknown) => number) | null ->() - /** * Compare two Temporal values of the same type, returning -1, 0, or 1. * - * Dispatch is keyed on `Symbol.toStringTag` (the brand already checked by - * `isTemporal`) rather than `a.constructor`, making it robust across realms - * and resistant to a shadowed `constructor` property. Types without a static - * `.compare` (e.g. `PlainMonthDay`) throw rather than fall back to string - * comparison, matching Temporal's design intent. + * The tags check type agreement. Types without a static `.compare` (e.g. + * `PlainMonthDay`) throw instead of using text order. * * Callers must ensure both arguments are Temporal objects; mixed types throw. */ export function compareTemporalValues(a: unknown, b: unknown): number { - const aTag = (a as Record)[Symbol.toStringTag] as string - const bTag = (b as Record)[Symbol.toStringTag] as string + const aTag = (a as Record)[Symbol.toStringTag] + const bTag = (b as Record)[Symbol.toStringTag] if (aTag !== bTag) { throw new TypeError( devBuild() && process.env.NODE_ENV !== `production` @@ -320,17 +319,10 @@ export function compareTemporalValues(a: unknown, b: unknown): number { : codedMessage(167, { aTag, bTag }), ) } - let compare = temporalCompareByTag.get(aTag) - if (compare === undefined) { - const fn = ( - (a as { constructor: unknown }).constructor as { - compare?: (x: unknown, y: unknown) => number - } - ).compare - compare = typeof fn === `function` ? fn : null - temporalCompareByTag.set(aTag, compare) - } - if (compare === null) { + const compare = ( + a as { constructor: { compare?: (x: unknown, y: unknown) => number } } + ).constructor.compare + if (typeof compare !== `function`) { throw new TypeError( devBuild() && process.env.NODE_ENV !== `production` ? `${aTag} has no defined ordering` diff --git a/packages/db/tests/basic-index-work.test.ts b/packages/db/tests/basic-index-work.test.ts index 438a9bfede..c513393795 100644 --- a/packages/db/tests/basic-index-work.test.ts +++ b/packages/db/tests/basic-index-work.test.ts @@ -1,7 +1,35 @@ import { describe, expect, it, vi } from 'vitest' +import { Temporal } from 'temporal-polyfill' import { BasicIndex } from '../src/indexes/basic-index' import { PropRef } from '../src/query/ir' +describe(`BasicIndex ordered read work`, () => { + it(`does not reserialize admitted Temporal values while reading`, () => { + const index = new BasicIndex(1, new PropRef([`value`])) + for (let key = 0; key < 64; key++) { + index.add(key, { + value: Temporal.Instant.fromEpochNanoseconds(BigInt(key)), + }) + } + + const toString = vi.spyOn(Temporal.Instant.prototype, `toString`) + let range = new Set() + let ordered: Array = [] + let stringifyCalls = 0 + try { + range = index.rangeQuery({}) + ordered = index.takeFromStart(64) + stringifyCalls = toString.mock.calls.length + } finally { + toString.mockRestore() + } + + expect(range.size).toBe(64) + expect(ordered).toEqual(Array.from({ length: 64 }, (_, key) => key)) + expect(stringifyCalls).toBe(0) + }) +}) + describe(`BasicIndex removal work`, () => { it.each([1, 4])( `searches only the comparator group of size %s`, diff --git a/packages/db/tests/btree-index-undefined-values.test.ts b/packages/db/tests/btree-index-undefined-values.test.ts index 252737e0ab..df46e80fa3 100644 --- a/packages/db/tests/btree-index-undefined-values.test.ts +++ b/packages/db/tests/btree-index-undefined-values.test.ts @@ -110,84 +110,15 @@ describe(`BTreeIndex - Issue #1186: Infinite loop with undefined indexed values` }) /** - * Direct unit tests for BTreeIndex undefined value handling. - * Tests the sentinel mechanism that distinguishes "start from beginning" - * (no from parameter) from "the key is literally undefined". + * Direct BTreeIndex tests for observations outside the direct-index oracle: + * internal ordering and storage, custom-comparator arguments, and the public + * live-query path above. */ describe(`BTreeIndex - undefined value handling`, () => { function createIndex(propName: string): BTreeIndex { return new BTreeIndex(1, new PropRef([propName]), `test_index`) } - describe(`take vs takeFromStart`, () => { - it(`should distinguish take(n, undefined) from takeFromStart(n)`, () => { - const index = createIndex(`value`) - index.add(`a`, { value: undefined }) - index.add(`b`, { value: 1 }) - index.add(`c`, { value: 2 }) - - const fromStart = index.takeFromStart(3) - expect(fromStart).toHaveLength(3) - - // take(n, undefined) returns items AFTER undefined (smallest value) - const afterUndefined = index.take(3, undefined) - expect(afterUndefined).toEqual([`b`, `c`]) - }) - - it(`should distinguish takeReversed(n, undefined) from takeReversedFromEnd(n)`, () => { - const index = createIndex(`value`) - index.add(`a`, { value: undefined }) - index.add(`b`, { value: 1 }) - index.add(`c`, { value: 2 }) - - const fromEnd = index.takeReversedFromEnd(3) - expect(fromEnd).toHaveLength(3) - - // Since undefined is smallest, nothing comes before it - const beforeUndefined = index.takeReversed(3, undefined) - expect(beforeUndefined).toHaveLength(0) - }) - }) - - describe(`multiple undefined values`, () => { - it(`should store and retrieve multiple items with undefined indexed values`, () => { - const index = createIndex(`priority`) - index.add(`a`, { priority: undefined }) - index.add(`b`, { priority: undefined }) - index.add(`c`, { priority: undefined }) - - expect(index.takeFromStart(10)).toEqual([`a`, `b`, `c`]) - - const undefinedItems = index.equalityLookup(undefined) - expect(undefinedItems.size).toBe(3) - }) - - it(`should correctly remove items with undefined values`, () => { - const index = createIndex(`priority`) - index.add(`a`, { priority: undefined }) - index.add(`b`, { priority: undefined }) - - index.remove(`a`, { priority: undefined }) - - const remaining = index.equalityLookup(undefined) - expect(remaining.size).toBe(1) - expect(remaining).toContain(`b`) - }) - - it(`should iterate past multiple undefined values correctly`, () => { - const index = createIndex(`value`) - index.add(`undef1`, { value: undefined }) - index.add(`undef2`, { value: undefined }) - index.add(`undef3`, { value: undefined }) - index.add(`one`, { value: 1 }) - index.add(`two`, { value: 2 }) - - // Start from undefined, should get items after all undefined values - const afterUndefined = index.take(10, undefined) - expect(afterUndefined).toEqual([`one`, `two`]) - }) - }) - describe(`ordering of undefined relative to other values`, () => { it(`should sort undefined before numbers with default comparator`, () => { const index = createIndex(`value`) @@ -209,82 +140,6 @@ describe(`BTreeIndex - undefined value handling`, () => { expect(orderedEntriesArray(index)[0]![0]).toBe(undefined) }) - - it(`should handle mixed undefined and null values`, () => { - const index = createIndex(`value`) - index.add(`null1`, { value: null }) - index.add(`undef1`, { value: undefined }) - index.add(`num1`, { value: 1 }) - - const nullItems = index.equalityLookup(null) - const undefItems = index.equalityLookup(undefined) - - expect(nullItems.size).toBe(1) - expect(nullItems).toContain(`null1`) - expect(undefItems.size).toBe(1) - expect(undefItems).toContain(`undef1`) - }) - }) - - describe(`rangeQuery with undefined bounds`, () => { - it(`should distinguish explicit undefined from vs no from`, () => { - const index = createIndex(`value`) - index.add(`undef`, { value: undefined }) - index.add(`one`, { value: 1 }) - index.add(`two`, { value: 2 }) - index.add(`three`, { value: 3 }) - - const withExplicitFrom = index.rangeQuery({ - from: undefined, - to: 2, - fromInclusive: true, - toInclusive: true, - }) - expect(withExplicitFrom.size).toBe(3) - expect(withExplicitFrom).toContain(`undef`) - expect(withExplicitFrom).toContain(`one`) - expect(withExplicitFrom).toContain(`two`) - - const withoutFrom = index.rangeQuery({ to: 2, toInclusive: true }) - expect(withoutFrom.size).toBe(3) - }) - - it(`should not drop the minimum key when an upper-only range is exclusive on the (absent) lower bound`, () => { - // When no `from` bound is provided, `fromInclusive` must not cause the - // smallest key to be excluded: there is no lower bound to exclude - // against. Only an explicitly provided exclusive lower bound should - // drop its boundary value. - const index = createIndex(`value`) - index.add(`a`, { value: 1 }) - index.add(`b`, { value: 5 }) - index.add(`c`, { value: 10 }) - - const result = index.rangeQuery({ to: 10, fromInclusive: false }) - - expect(result.size).toBe(3) - expect(result).toContain(`a`) - expect(result).toContain(`b`) - expect(result).toContain(`c`) - }) - - it(`should handle range query from undefined to undefined`, () => { - const index = createIndex(`value`) - index.add(`a`, { value: undefined }) - index.add(`b`, { value: undefined }) - index.add(`c`, { value: 1 }) - - const justUndefined = index.rangeQuery({ - from: undefined, - to: undefined, - fromInclusive: true, - toInclusive: true, - }) - - expect(justUndefined.size).toBe(2) - expect(justUndefined).toContain(`a`) - expect(justUndefined).toContain(`b`) - expect(justUndefined).not.toContain(`c`) - }) }) describe(`custom comparator with undefined values`, () => { @@ -319,90 +174,6 @@ describe(`BTreeIndex - undefined value handling`, () => { }) }) - describe(`equalityLookup with undefined`, () => { - it(`should find items with undefined indexed value`, () => { - const index = createIndex(`status`) - index.add(`a`, { status: `active` }) - index.add(`b`, { status: undefined }) - index.add(`c`, { status: `inactive` }) - index.add(`d`, { status: undefined }) - - const undefinedItems = index.equalityLookup(undefined) - expect(undefinedItems.size).toBe(2) - expect(undefinedItems).toContain(`b`) - expect(undefinedItems).toContain(`d`) - }) - - it(`should return empty set when no undefined values exist`, () => { - const index = createIndex(`value`) - index.add(`a`, { value: 1 }) - index.add(`b`, { value: 2 }) - - expect(index.equalityLookup(undefined).size).toBe(0) - }) - }) - - describe(`inArrayLookup with undefined`, () => { - it(`should handle undefined in inArrayLookup`, () => { - const index = createIndex(`value`) - index.add(`undef`, { value: undefined }) - index.add(`one`, { value: 1 }) - index.add(`two`, { value: 2 }) - index.add(`three`, { value: 3 }) - - const result = index.inArrayLookup([undefined, 1]) - expect(result.size).toBe(2) - expect(result).toContain(`undef`) - expect(result).toContain(`one`) - }) - - it(`should handle inArrayLookup with only undefined`, () => { - const index = createIndex(`value`) - index.add(`undef1`, { value: undefined }) - index.add(`undef2`, { value: undefined }) - index.add(`one`, { value: 1 }) - - const result = index.inArrayLookup([undefined]) - expect(result.size).toBe(2) - expect(result).toContain(`undef1`) - expect(result).toContain(`undef2`) - }) - }) - - describe(`update operation with undefined`, () => { - it(`should update item from defined to undefined value`, () => { - const index = createIndex(`value`) - index.add(`a`, { value: 1 }) - - index.update(`a`, { value: 1 }, { value: undefined }) - - expect(index.equalityLookup(1).size).toBe(0) - expect(index.equalityLookup(undefined).size).toBe(1) - expect(index.equalityLookup(undefined)).toContain(`a`) - }) - - it(`should update item from undefined to defined value`, () => { - const index = createIndex(`value`) - index.add(`a`, { value: undefined }) - - index.update(`a`, { value: undefined }, { value: 1 }) - - expect(index.equalityLookup(undefined).size).toBe(0) - expect(index.equalityLookup(1).size).toBe(1) - expect(index.equalityLookup(1)).toContain(`a`) - }) - - it(`should update item from undefined to undefined (same value)`, () => { - const index = createIndex(`value`) - index.add(`a`, { value: undefined }) - - index.update(`a`, { value: undefined }, { value: undefined }) - - expect(index.equalityLookup(undefined).size).toBe(1) - expect(index.equalityLookup(undefined)).toContain(`a`) - }) - }) - describe(`valueMapData getter`, () => { it(`should return undefined keys denormalized, not as sentinel`, () => { const index = createIndex(`value`) diff --git a/packages/db/tests/btree-index-work.test.ts b/packages/db/tests/btree-index-work.test.ts index d907df5694..ee15d5e808 100644 --- a/packages/db/tests/btree-index-work.test.ts +++ b/packages/db/tests/btree-index-work.test.ts @@ -1,7 +1,44 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' +import { Temporal } from 'temporal-polyfill' import { BTreeIndex } from '../src/indexes/btree-index.js' import { PropRef } from '../src/query/ir.js' +describe(`BTree comparator-tie update work`, () => { + it(`does not copy every owner when an exact value returns to a tie`, () => { + const instant = Temporal.Instant.fromEpochNanoseconds(0n) + const first = instant.toZonedDateTimeISO(`UTC`) + const second = instant.toZonedDateTimeISO(`+14:00`) + const additions = [16, 256].map((size) => { + const index = new BTreeIndex(1, new PropRef([`value`])) + for (let key = 0; key < size; key++) { + index.add(key, { value: first }) + } + index.add(-1, { value: second }) + index.remove(-1, { value: second }) + + const originalAdd = Set.prototype.add + const add = vi.spyOn(Set.prototype, `add`).mockImplementation(function ( + this: Set, + value: unknown, + ) { + return originalAdd.call(this, value) + }) + let calls = 0 + try { + index.add(-1, { value: second }) + calls = add.mock.calls.length + } finally { + add.mockRestore() + } + expect(index.equalityLookup(first).size).toBe(size) + expect(index.equalityLookup(second)).toEqual(new Set([-1])) + expect(index.takeFromStart(1)).toEqual([-1]) + return calls + }) + expect(additions[1]).toBeLessThanOrEqual(additions[0]! + 4) + }) +}) + describe(`BTree exact-bucket ownership work`, () => { it.each( [300, 100000].flatMap((size) => diff --git a/packages/db/tests/change-event-history-oracle.test.ts b/packages/db/tests/change-event-history-oracle.test.ts index 005b450e22..719cdb404b 100644 --- a/packages/db/tests/change-event-history-oracle.test.ts +++ b/packages/db/tests/change-event-history-oracle.test.ts @@ -1,4 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' +import { Temporal } from 'temporal-polyfill' import { describe, expect, it, vi } from 'vitest' import { BTreeIndex, @@ -572,6 +573,89 @@ async function verifyGeneratedHistory({ } describe(`change-event history oracle`, () => { + // Published #1519 keeps equivalent Duration forms distinct for query `eq`. + // Under #1901's change-message reconstruction law, a source replacement + // that changes the Duration's intrinsic fields must reach the consumer's + // mirror as well as the public row. This two-step history inserts one second, + // then replaces it with 1000 milliseconds. The reference Map applies that + // valid write despite the shared display text; at the update commit, both + // public observations must reflect the second value. + // This witness covers ordinary same-realm Durations, not altered objects or + // a requirement to suppress events for a separately allocated equal copy. + it(`publishes a Duration field change to the Collection and change-message consumer`, async () => { + interface DurationRow extends Record { + id: number + duration: Temporal.Duration + } + let sync!: Parameters[`sync`]>[0] + const collection = createCollection({ + id: `duration-field-change-history`, + getKey: (row) => row.id, + startSync: true, + sync: { + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + + await runWithCleanup( + async () => { + await collection.stateWhenReady() + const first = { + id: 1, + duration: Temporal.Duration.from({ seconds: 1 }), + } + const second = { + id: 1, + duration: Temporal.Duration.from({ milliseconds: 1000 }), + } + expect(first.duration.toString()).toBe(second.duration.toString()) + sync.begin() + sync.write({ type: `insert`, value: first }) + expect(sync.commit()).toBe(true) + + // The model applies each valid source write to a plain keyed Map. + // A subscriber maintains an independent mirror from public messages. + const expected = new Map([[1, first]]) + const mirror = new Map(expected) + const received: Array> = [] + const subscription = collection.subscribeChanges( + (changes) => { + received.push(...changes) + for (const change of changes) { + if (change.type === `delete`) mirror.delete(change.key) + else mirror.set(change.key, change.value) + } + }, + { includeInitialState: false }, + ) + try { + expected.set(1, second) + sync.begin() + sync.write({ type: `update`, value: second }) + expect(sync.commit()).toBe(true) + + const expectedFields = [0, 1000] + const fields = (row: DurationRow | undefined) => [ + row?.duration.seconds, + row?.duration.milliseconds, + ] + expect(fields(expected.get(1))).toEqual(expectedFields) + expect(fields(collection.state.get(1))).toEqual(expectedFields) + expect(received.map((message) => message.type)).toEqual([`update`]) + expect(fields(received[0]?.previousValue)).toEqual([1, 0]) + expect(fields(received[0]?.value)).toEqual(expectedFields) + expect(fields(mirror.get(1))).toEqual(expectedFields) + } finally { + subscription.unsubscribe() + } + }, + () => collection.cleanup(), + ) + }) + it(`preserves each key's causal messages within a deferred callback`, async () => { const rounds: ReadonlyArray> = [ [ diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index db5ffee2b7..d08920b07d 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -23,7 +23,6 @@ import { expectIndexUsage, indexedKeysSet, orderedEntriesArray, - orderedEntriesArrayReversed, stripVirtualProps, valueMapData, withIndexTracking, @@ -1738,12 +1737,7 @@ describe(`Collection Indexes`, () => { const index = groupedCollection.createIndex((row) => row.value) expect(index.takeFromStart(2)).toEqual([`first`, `second`]) - expect(orderedEntriesArray(index)[0]?.[1]).toEqual( - new Set([`first`, `second`]), - ) - expect(orderedEntriesArrayReversed(index)[0]?.[1]).toEqual( - new Set([`first`, `second`]), - ) + expect(index.takeReversedFromEnd(2)).toEqual([`second`, `first`]) }) it(`should return all matching rows for a range predicate on a custom-comparator index`, async () => { diff --git a/packages/db/tests/comparison-oracle.property.test.ts b/packages/db/tests/comparison-oracle.property.test.ts index 55ad28c4d2..1f8277456f 100644 --- a/packages/db/tests/comparison-oracle.property.test.ts +++ b/packages/db/tests/comparison-oracle.property.test.ts @@ -1,19 +1,26 @@ import { describe, expect, it } from 'vitest' import { fc, test as fcTest } from '@fast-check/vitest' +import { Temporal } from 'temporal-polyfill' import { areValuesEqual, ascComparator, + defaultComparator, descComparator, makeComparator, normalizeValue, } from '../src/utils/comparison' +import { DEFAULT_COMPARE_OPTIONS } from '../src/utils' import type { CompareOptions } from '../src/query/builder/types' /** * Comparison defines the order and equality domains used by queries and * indexes. The laws are reflexivity, deterministic sign, antisymmetry, and * transitivity under one resolved option set. Null placement, direction, and - * lexical/locale string modes are independent axes. + * lexical/locale string modes are independent axes. In the bounded default + * ascending domain below, NaN and invalid Dates share one non-finite order + * position and follow finite values; null remains first. The published NaN + * ordering change in `packages/db/CHANGELOG.md` (#1582) establishes that law + * for ordinary numbers and Dates in the transferred unit-test domain. * * Authority: `OrderByOptions` and `CompareOptions` in the query builder's * option types declare these modes. The index comparator contract accepts @@ -21,10 +28,14 @@ import type { CompareOptions } from '../src/query/builder/types' * * The model uses direct primitive, Date, array, and byte comparisons from the * declared domain. It excludes unrelated object identities because production - * intentionally assigns those a creation-order ID. Equality laws separately - * cover the normalized value classes and changed-byte controls. Comparator - * signs reject NaN and non-number results before reduction. Signed infinity - * is a valid comparator result, including for custom string collation. + * intentionally assigns those a creation-order ID. The non-finite model uses + * input tags and finite epoch-day values, rather than the production + * unorderable classifier. It covers number-only and Date-only sorts with + * optional null; it does not claim mixed-type order or query/index paths. + * Equality laws separately cover normalized value classes and changed-byte + * controls. Comparator signs reject NaN and non-number results before + * reduction. Signed infinity is a valid comparator result, including for + * custom string collation. * * Binary normalization has a narrower, established contract: equal byte * contents produce one Map key, different contents remain distinct, and a @@ -200,8 +211,9 @@ function comparisonProperty< // locale/lexical, and custom-collation cases reconstruct those boundaries. // Array length 0/1 and binary length 0/1/128/129 are marginal cases. Plain // objects are outside this order model because their IDs depend on creation -// order. Mixed-type order and NaN inputs are outside this bounded grammar; an -// invalid comparator result is rejected by the control below. +// order. Mixed-type order is outside this bounded grammar. A tagged grammar +// below admits NaN and invalid Date inputs; an invalid comparator result is +// rejected by the control below. const arbitraryComparablePrimitive = fc.oneof( fc.string(), fc.integer(), @@ -211,6 +223,108 @@ const arbitraryComparablePrimitive = fc.oneof( const arbitraryDate = fc.date({ noInvalidDate: true }) +type NonFiniteFamily = `number` | `date` +type NonFiniteInput = { + family: NonFiniteFamily + finiteDays: Array + unorderedCount: number + includeNull: boolean +} +type TaggedOrderValue = { + id: string + value: number | Date | null + category: `null` | `finite` | `unordered` + order: number +} + +// The generator's tags are reference facts: finite epoch days have ordinary +// numeric order, null precedes them, and the family's non-finite values follow +// them. Constructing expected signs from these tags avoids asking production +// whether a Date or number is unorderable. Distinct IDs preserve duplicate +// NaNs/invalid Dates in the sorted observation. +function taggedOrderValues(input: NonFiniteInput): Array { + const unordered = () => + input.family === `number` ? Number.NaN : new Date(Number.NaN) + const finite = input.finiteDays.map((day, index) => ({ + id: `finite:${index}`, + value: input.family === `number` ? day : new Date(day * 86_400_000), + category: `finite` as const, + order: day, + })) + const rows: Array = [ + finite[0]!, + { id: `unordered:0`, value: unordered(), category: `unordered`, order: 0 }, + ...finite.slice(1), + ...Array.from({ length: input.unorderedCount - 1 }, (_, index) => ({ + id: `unordered:${index + 1}`, + value: unordered(), + category: `unordered` as const, + order: 0, + })), + ] + if (input.includeNull) { + rows.splice(2, 0, { id: `null`, value: null, category: `null`, order: 0 }) + } + return rows +} + +function taggedOrderSign(a: TaggedOrderValue, b: TaggedOrderValue): -1 | 0 | 1 { + const rank = { null: 0, finite: 1, unordered: 2 } + const difference = rank[a.category] - rank[b.category] + const valueDifference = + difference || (a.category === `finite` ? a.order - b.order : 0) + return valueDifference < 0 ? -1 : valueDifference > 0 ? 1 : 0 +} + +// Production observes direct comparator returns and the order of a stable +// JavaScript sort using the exported default comparator. The comparison runs +// immediately after each call, so it detects wrong signs and lost or reordered +// duplicate non-finite rows at the exact sort checkpoint. +function assertNonFiniteOrder( + input: NonFiniteInput, + compare: (a: number | Date | null, b: number | Date | null) => number = ( + a, + b, + ) => ascComparator(a, b, DEFAULT_COMPARE_OPTIONS), + sortCompare: ( + a: number | Date | null, + b: number | Date | null, + ) => number = defaultComparator, +): void { + const rows = taggedOrderValues(input) + for (const a of rows) { + for (const b of rows) { + const result = compare(a.value, b.value) + expect(sign(result)).toBe(taggedOrderSign(a, b)) + // Preserve the legacy exact-zero assertion for NaN/NaN, including its + // distinction between positive and negative zero in Vitest. + if (a.category === `unordered` && b.category === `unordered`) { + expect(result).toBe(0) + } + } + } + const actual = [...rows].sort((a, b) => sortCompare(a.value, b.value)) + const expected = [...rows].sort(taggedOrderSign) + expect(actual.map((row) => row.id)).toEqual(expected.map((row) => row.id)) +} + +// Legal inputs contain 1..5 distinct finite numbers or valid epoch-day Dates, +// one or two NaNs/invalid Dates, and optional null. A singleton finite value, +// duplicate non-finite values, negative days, and null presence are margins. +// Removing the family axis misses invalid Dates; removing the duplicate count +// misses equal non-finite rows; removing null misses its default placement. +// Mixed families, Temporal values, symbols, and invalid option objects are not +// generated. The fixed examples below reconstruct the legacy sort histories. +const arbitraryNonFiniteInput: fc.Arbitrary = fc.record({ + family: fc.constantFrom(`number` as const, `date` as const), + finiteDays: fc.uniqueArray(fc.integer({ min: -20_000, max: 20_000 }), { + minLength: 1, + maxLength: 5, + }), + unorderedCount: fc.integer({ min: 1, max: 2 }), + includeNull: fc.boolean(), +}) + const arbitraryComparableArray = fc.array(arbitraryComparablePrimitive, { maxLength: 5, }) @@ -677,6 +791,100 @@ describe(`ascComparator property-based tests`, () => { }) }) +describe(`NaN and invalid Date default-order refinement`, () => { + it(`reconstructs the finite, duplicate NaN, null, and invalid Date witnesses`, () => { + assertNonFiniteOrder({ + family: `number`, + finiteDays: [3, 1, 5], + unorderedCount: 2, + includeNull: false, + }) + assertNonFiniteOrder({ + family: `number`, + finiteDays: [5, 1], + unorderedCount: 1, + includeNull: true, + }) + assertNonFiniteOrder({ + family: `date`, + finiteDays: [19_358], // 2023-01-01, the legacy valid-Date operand. + unorderedCount: 1, + includeNull: false, + }) + }) + + comparisonProperty([arbitraryNonFiniteInput])( + `default comparator refines tagged finite and non-finite order`, + (input) => assertNonFiniteOrder(input), + ) + + it(`rejects a wrong comparison that ties non-finite and finite values`, () => { + for (const family of [`number`, `date`] as const) { + let reachedFault = false + const faultyComparator = ( + a: number | Date | null, + b: number | Date | null, + ) => { + const faultApplies = + family === `number` + ? typeof a === `number` && + Number.isNaN(a) && + typeof b === `number` && + Number.isFinite(b) + : a instanceof Date && + Number.isNaN(a.getTime()) && + b instanceof Date && + Number.isFinite(b.getTime()) + if (faultApplies) { + reachedFault = true + return 0 + } + return ascComparator(a, b, DEFAULT_COMPARE_OPTIONS) + } + expect(() => + assertNonFiniteOrder( + { + family, + finiteDays: [1], + unorderedCount: 1, + includeNull: false, + }, + faultyComparator, + ), + ).toThrowError() + expect(reachedFault).toBe(true) + } + }) + + it(`rejects a default sort that ties NaN to finite values`, () => { + let reachedFault = false + const faultySort = (a: number | Date | null, b: number | Date | null) => { + if (typeof a === `number` && Number.isNaN(a)) { + reachedFault = true + return 0 + } + if (typeof b === `number` && Number.isNaN(b)) { + reachedFault = true + return 0 + } + return defaultComparator(a, b) + } + expect(() => + assertNonFiniteOrder( + { + family: `number`, + finiteDays: [3, 1, 5], + unorderedCount: 2, + includeNull: false, + }, + (a, b) => ascComparator(a, b, DEFAULT_COMPARE_OPTIONS), + faultySort, + ), + ).toThrowError() + expect(reachedFault).toBe(true) + }) +}) + describe(`descComparator property-based tests`, () => { comparisonProperty([arbitrarySameTypePair])( `descComparator reverses ascComparator ordering (same types)`, @@ -831,17 +1039,6 @@ describe(`normalizeValue property-based tests`, () => { expect(normalizeValue(normalized)).not.toBe(normalized) }, ) - - fcTest(`reads binary keys from indexed bytes, not custom iteration`, () => { - const bytes = new Uint8Array([2]) - Object.defineProperty(bytes, Symbol.iterator, { - value: function* () { - yield 1 - }, - }) - - expect(normalizeValue(bytes)).toBe(normalizeValue(new Uint8Array([2]))) - }) }) describe(`areValuesEqual property-based tests`, () => { @@ -882,6 +1079,230 @@ describe(`areValuesEqual property-based tests`, () => { ) }) +/** + * Temporal ordering uses calendar position or instant. The published #1519 + * contract gives PlainDate calendar order + * and ZonedDateTime instant order even when zones differ. These models use + * supplied ISO date fields and epoch-nanosecond offsets; neither asks the + * production comparator or Temporal's static compare for an expected sign. + * + * Legal inputs are ISO PlainDates from years 2000..2030 with days 1..28 and + * ZonedDateTimes within two seconds of a fixed instant in four offset zones. + * Equal-value branches construct separate objects, and the zoned branch forces + * different zones at the same instant. The production path is the exported + * `ascComparator` under default options; each return is the observation + * checkpoint. This lane makes no indexed or public-query claim. + */ +type IsoDateFields = { year: number; month: number; day: number } +type PlainDateOrderInput = { left: IsoDateFields; right: IsoDateFields } + +function modelPlainDateSign( + left: IsoDateFields, + right: IsoDateFields, +): -1 | 0 | 1 { + for (const field of [`year`, `month`, `day`] as const) { + if (left[field] < right[field]) return -1 + if (left[field] > right[field]) return 1 + } + return 0 +} + +function assertPlainDateOrder(input: PlainDateOrderInput): void { + const left = new Temporal.PlainDate( + input.left.year, + input.left.month, + input.left.day, + ) + const right = new Temporal.PlainDate( + input.right.year, + input.right.month, + input.right.day, + ) + const expected = modelPlainDateSign(input.left, input.right) + const actual = ascComparator(left, right, DEFAULT_COMPARE_OPTIONS) + const reverse = ascComparator(right, left, DEFAULT_COMPARE_OPTIONS) + expect(sign(actual)).toBe(expected) + expect(sign(reverse)).toBe(expected === 0 ? 0 : -expected) + if (expected === 0) { + // The old unit block asserted an exact zero for two distinct instances. + expect(left).not.toBe(right) + expect(actual).toBe(0) + expect(reverse).toBe(0) + } +} + +const arbitraryIsoDateFields: fc.Arbitrary = fc.record({ + year: fc.integer({ min: 2000, max: 2030 }), + month: fc.integer({ min: 1, max: 12 }), + day: fc.integer({ min: 1, max: 28 }), +}) +const arbitraryPlainDateOrderInput: fc.Arbitrary = + fc.oneof( + fc.record({ + left: arbitraryIsoDateFields, + right: arbitraryIsoDateFields, + }), + arbitraryIsoDateFields.map((fields) => ({ left: fields, right: fields })), + ) + +type OffsetZone = `+00:00` | `+05:30` | `-05:00` | `+14:00` +type ZonedDateTimeOrderInput = { + leftNanoOffset: number + rightNanoOffset: number + leftZone: OffsetZone + rightZone: OffsetZone +} + +// The base instant reconstructs the old 06:30 UTC / 12:00 +05:30 witness. +// Offsets are safe integers, so their ordinary numeric order is the reference +// for the generated epoch nanoseconds. +const zonedBaseEpochNanoseconds = + BigInt(Date.UTC(2024, 0, 15, 6, 30)) * 1_000_000n + +function modelNanosecondSign(left: number, right: number): -1 | 0 | 1 { + return left < right ? -1 : left > right ? 1 : 0 +} + +function zonedValue(nanoOffset: number, zone: OffsetZone) { + return Temporal.Instant.fromEpochNanoseconds( + zonedBaseEpochNanoseconds + BigInt(nanoOffset), + ).toZonedDateTimeISO(zone) +} + +function assertZonedDateTimeOrder( + input: ZonedDateTimeOrderInput, + compare: (left: unknown, right: unknown) => number = (left, right) => + ascComparator(left, right, DEFAULT_COMPARE_OPTIONS), +): void { + const left = zonedValue(input.leftNanoOffset, input.leftZone) + const right = zonedValue(input.rightNanoOffset, input.rightZone) + const expected = modelNanosecondSign( + input.leftNanoOffset, + input.rightNanoOffset, + ) + const actual = compare(left, right) + const reverse = compare(right, left) + expect(sign(actual)).toBe(expected) + expect(sign(reverse)).toBe(expected === 0 ? 0 : -expected) + if (expected === 0) { + expect(actual).toBe(0) + expect(reverse).toBe(0) + if (input.leftZone !== input.rightZone) { + // A text-order mutant sees different strings for one model instant. + expect(left.toString()).not.toBe(right.toString()) + } + } +} + +const arbitraryNanoOffset = fc.integer({ + min: -1_999_999_999, + max: 1_999_999_999, +}) +const arbitraryOffsetZone: fc.Arbitrary = fc.constantFrom( + `+00:00`, + `+05:30`, + `-05:00`, + `+14:00`, +) +const arbitraryZonedDateTimeOrderInput: fc.Arbitrary = + fc.oneof( + fc.record({ + leftNanoOffset: arbitraryNanoOffset, + rightNanoOffset: arbitraryNanoOffset, + leftZone: arbitraryOffsetZone, + rightZone: arbitraryOffsetZone, + }), + fc + .tuple(arbitraryNanoOffset, arbitraryOffsetZone, arbitraryOffsetZone) + .filter(([, leftZone, rightZone]) => leftZone !== rightZone) + .map(([nanoOffset, leftZone, rightZone]) => ({ + leftNanoOffset: nanoOffset, + rightNanoOffset: nanoOffset, + leftZone, + rightZone, + })), + fc + .tuple( + arbitraryNanoOffset, + fc.boolean(), + arbitraryOffsetZone, + arbitraryOffsetZone, + ) + .map(([nanoOffset, forward, leftZone, rightZone]) => ({ + leftNanoOffset: nanoOffset, + rightNanoOffset: nanoOffset + (forward ? 1 : -1), + leftZone, + rightZone, + })), + ) + +describe(`Temporal default-order refinement`, () => { + it(`reconstructs the three PlainDate and one ZonedDateTime assertions`, () => { + const jan = { year: 2024, month: 1, day: 1 } + const june = { year: 2024, month: 6, day: 1 } + assertPlainDateOrder({ + left: { year: 2023, month: 12, day: 28 }, + right: jan, + }) + assertPlainDateOrder({ left: jan, right: june }) + assertPlainDateOrder({ + left: jan, + right: { year: 2024, month: 1, day: 2 }, + }) + assertPlainDateOrder({ left: jan, right: jan }) + assertZonedDateTimeOrder({ + leftNanoOffset: 0, + rightNanoOffset: 0, + leftZone: `+00:00`, + rightZone: `+05:30`, + }) + assertZonedDateTimeOrder({ + leftNanoOffset: 0, + rightNanoOffset: 1, + leftZone: `+05:30`, + rightZone: `+00:00`, + }) + }) + + comparisonProperty([arbitraryPlainDateOrderInput])( + `ISO PlainDate order refines supplied calendar fields`, + assertPlainDateOrder, + ) + + comparisonProperty([arbitraryZonedDateTimeOrderInput])( + `ZonedDateTime order refines supplied instants across offset zones`, + assertZonedDateTimeOrder, + ) + + it(`rejects text order for one instant shown in two zones`, () => { + const sample: ZonedDateTimeOrderInput = { + leftNanoOffset: 0, + rightNanoOffset: 0, + leftZone: `+00:00`, + rightZone: `+05:30`, + } + const lexical = (left: unknown, right: unknown) => + String(left).localeCompare(String(right)) + expect(() => assertZonedDateTimeOrder(sample, lexical)).toThrow() + }) + + it(`rejects millisecond truncation across adjacent instants`, () => { + const sample: ZonedDateTimeOrderInput = { + leftNanoOffset: 0, + rightNanoOffset: 1, + leftZone: `+05:30`, + rightZone: `+00:00`, + } + const truncated = (left: unknown, right: unknown) => { + const leftEpoch = (left as { epochNanoseconds: bigint }).epochNanoseconds + const rightEpoch = (right as { epochNanoseconds: bigint }) + .epochNanoseconds + return Number(leftEpoch / 1_000_000n - rightEpoch / 1_000_000n) + } + expect(() => assertZonedDateTimeOrder(sample, truncated)).toThrow() + }) +}) + if (requestedProperty !== undefined) { it(`selects a registered comparison oracle property`, () => { expect(requestedProperty).toMatch(/^\d+$/) diff --git a/packages/db/tests/comparison.test.ts b/packages/db/tests/comparison.test.ts index 870b23978b..7c60bc5d0c 100644 --- a/packages/db/tests/comparison.test.ts +++ b/packages/db/tests/comparison.test.ts @@ -1,72 +1,7 @@ import { describe, expect, it } from 'vitest' -import { Temporal } from 'temporal-polyfill' -import { - ascComparator, - compareValues, - defaultComparator, -} from '../src/utils/comparison' +import { ascComparator, compareValues } from '../src/utils/comparison' import { DEFAULT_COMPARE_OPTIONS } from '../src/utils' -describe(`ascComparator - PostgreSQL float semantics for NaN`, () => { - const opts = DEFAULT_COMPARE_OPTIONS // nulls: `first` - - it(`orders NaN greater than every number`, () => { - expect(ascComparator(NaN, 5, opts)).toBeGreaterThan(0) - expect(ascComparator(5, NaN, opts)).toBeLessThan(0) - }) - - it(`treats NaN as equal to NaN`, () => { - expect(ascComparator(NaN, NaN, opts)).toBe(0) - }) - - it(`produces a stable total order with NaN sorting last`, () => { - const sorted = [3, NaN, 1, 5, NaN].sort((a, b) => defaultComparator(a, b)) - - expect(sorted.slice(0, 3)).toEqual([1, 3, 5]) - expect(sorted.slice(3).every((v) => Number.isNaN(v))).toBe(true) - }) - - it(`keeps null before non-null values regardless of NaN`, () => { - // nulls still sort first by default; NaN sorts last (greatest non-null) - const sorted = [5, NaN, null, 1].sort((a, b) => defaultComparator(a, b)) - - expect(sorted[0]).toBe(null) - expect(sorted[1]).toBe(1) - expect(sorted[2]).toBe(5) - expect(Number.isNaN(sorted[3])).toBe(true) - }) - - it(`orders an invalid Date greater than valid Dates`, () => { - const invalid = new Date(`not a date`) - const valid = new Date(`2023-01-01`) - - expect(ascComparator(invalid, valid, opts)).toBeGreaterThan(0) - expect(ascComparator(valid, invalid, opts)).toBeLessThan(0) - }) -}) - -describe(`ascComparator - Temporal values`, () => { - const opts = DEFAULT_COMPARE_OPTIONS - - it(`orders PlainDate values by calendar date`, () => { - const earlier = new Temporal.PlainDate(2024, 1, 1) - const later = new Temporal.PlainDate(2024, 6, 1) - expect(ascComparator(earlier, later, opts)).toBeLessThan(0) - expect(ascComparator(later, earlier, opts)).toBeGreaterThan(0) - expect( - ascComparator(earlier, new Temporal.PlainDate(2024, 1, 1), opts), - ).toBe(0) - }) - - it(`treats ZonedDateTime values at the same instant as equal regardless of zone`, () => { - // 2024-01-15T06:30Z and 2024-01-15T12:00+05:30 are the same instant; - // lexicographic toString comparison would order them differently. - const utc = Temporal.ZonedDateTime.from(`2024-01-15T06:30:00+00:00[+00:00]`) - const ist = Temporal.ZonedDateTime.from(`2024-01-15T12:00:00+05:30[+05:30]`) - expect(ascComparator(utc, ist, opts)).toBe(0) - }) -}) - describe(`ascComparator - symbols`, () => { const opts = DEFAULT_COMPARE_OPTIONS diff --git a/packages/db/tests/duration-query-equality-oracle.test.ts b/packages/db/tests/duration-query-equality-oracle.test.ts new file mode 100644 index 0000000000..1651188f88 --- /dev/null +++ b/packages/db/tests/duration-query-equality-oracle.test.ts @@ -0,0 +1,219 @@ +/** + * Query equality preserves each of a Temporal.Duration's ten fields. Changelog + * #1519 explicitly keeps equivalent Duration forms unequal under `eq`: one + * second and 1000 milliseconds display alike but have different fields. + * + * The independent model reads the ten public fields rather than formatting a + * Duration or importing query normalization. The fixed input grammar inserts + * one ordinary Duration row, then probes an equal copy and a changed value + * for every field. A two-row calendar-unit history checks that equality + * indexing remains available when Duration has no intrinsic order without + * `relativeTo`. Three further probes distinguish a same-display unit change, + * opposite signs, and a changed second nonzero field. + * Each direct BasicIndex/BTreeIndex exact lookup and each public Collection + * `eq` read before and after indexing must return the model's exact key set at + * that read. An index spy confirms the indexed public route ran. These checks + * make no claim about Duration ordering, other Temporal kinds, mutations, or + * live subscriber publication. + */ +import { expect, it, vi } from 'vitest' +import { Temporal } from 'temporal-polyfill' +import { createCollection } from '../src/collection/index.js' +import { BasicIndex } from '../src/indexes/basic-index.js' +import { BTreeIndex } from '../src/indexes/btree-index.js' +import { eq } from '../src/query/builder/functions.js' +import { PropRef } from '../src/query/ir.js' +import { mockSyncCollectionOptions } from './utils.js' + +const fields = [ + `years`, + `months`, + `weeks`, + `days`, + `hours`, + `minutes`, + `seconds`, + `milliseconds`, + `microseconds`, + `nanoseconds`, +] as const + +function modelEqual( + left: Temporal.Duration, + right: Temporal.Duration, +): boolean { + return fields.every((field) => left[field] === right[field]) +} + +type Case = { + name: string + stored: Temporal.Duration + probe: Temporal.Duration +} + +const sameDisplay: Case = { + name: `seconds versus milliseconds with identical display`, + stored: Temporal.Duration.from({ seconds: 1 }), + probe: Temporal.Duration.from({ milliseconds: 1000 }), +} +const oppositeSigns: Case = { + name: `opposite signs`, + stored: Temporal.Duration.from({ seconds: 1 }), + probe: Temporal.Duration.from({ seconds: -1 }), +} +const secondFieldChange: Case = { + name: `same first nonzero field with a changed second field`, + stored: Temporal.Duration.from({ hours: 1, minutes: 2 }), + probe: Temporal.Duration.from({ hours: 1, minutes: 3 }), +} + +const cases: ReadonlyArray = [ + ...fields.flatMap((field) => { + const stored = Temporal.Duration.from({ [field]: 1 }) + return [ + { + name: `${field} equal copy`, + stored, + probe: Temporal.Duration.from({ [field]: 1 }), + }, + { + name: `${field} changed`, + stored, + probe: Temporal.Duration.from({ [field]: 2 }), + }, + ] + }), + sameDisplay, + oppositeSigns, + secondFieldChange, +] + +function modelKeys({ stored, probe }: Case): Set { + return new Set(modelEqual(stored, probe) ? [`row`] : []) +} + +const indexTypes = [ + [`BasicIndex`, BasicIndex], + [`BTreeIndex`, BTreeIndex], +] as const + +// Every field changes the model's answer, while a separately constructed +// equal copy does not. Three pairs kill text, absolute-value, and first-field +// key mutants at the same exact-key-set comparison used below. +it(`calibrates all ten fields and three plausible wrong keys`, () => { + for (const [equal, changed] of Array.from({ length: 10 }, (_, index) => + cases.slice(index * 2, index * 2 + 2), + )) { + expect(modelEqual(equal!.stored, equal!.probe)).toBe(true) + expect(modelEqual(changed!.stored, changed!.probe)).toBe(false) + } + expect(String(sameDisplay.stored)).toBe(String(sameDisplay.probe)) + expect(modelEqual(sameDisplay.stored, sameDisplay.probe)).toBe(false) + const magnitudeKey = (value: Temporal.Duration) => + fields.map((field) => Math.abs(value[field])).join(`,`) + const firstNonzeroKey = (value: Temporal.Duration) => { + const field = fields.find((candidate) => value[candidate] !== 0)! + return `${field}:${value[field]}` + } + const wrongKeys = ( + testCase: Case, + key: (value: Temporal.Duration) => string, + ) => new Set(key(testCase.stored) === key(testCase.probe) ? [`row`] : []) + for (const [testCase, key] of [ + [sameDisplay, String], + [oppositeSigns, magnitudeKey], + [secondFieldChange, firstNonzeroKey], + ] as const) { + expect(() => + expect(wrongKeys(testCase, key)).toEqual(modelKeys(testCase)), + ).toThrow() + } +}) + +it.each(indexTypes)( + `%s exact lookup follows Duration fields`, + (_name, IndexType) => { + for (const { name, stored, probe } of cases) { + const index = new IndexType(1, new PropRef([`value`])) + index.add(`row`, { value: stored }) + expect(index.lookup(`eq`, probe), name).toEqual( + modelKeys({ name, stored, probe }), + ) + } + }, +) + +it.each(indexTypes)( + `public eq agrees before and after %s indexing`, + async (_name, IndexType) => { + for (const { name, stored, probe } of cases) { + const source = createCollection( + mockSyncCollectionOptions({ + id: `duration-query-equality-${name}`, + getKey: (row: { id: string; value: Temporal.Duration }) => row.id, + initialData: [{ id: `row`, value: stored }], + autoIndex: `off`, + defaultIndexType: IndexType, + }), + ) + try { + await source.stateWhenReady() + const where = eq(new PropRef([`value`]), probe) + const expected = [...modelKeys({ name, stored, probe })] + const readKeys = () => + source.currentStateAsChanges({ where })!.map((change) => change.key) + expect(readKeys(), `${name}: scan`).toEqual(expected) + const index = source.createIndex((row) => row.value) + const lookup = vi.spyOn(index, `lookup`) + expect(readKeys(), `${name}: indexed`).toEqual(expected) + expect(lookup, `${name}: index path`).toHaveBeenCalled() + } finally { + await source.cleanup() + } + } + }, +) + +// The model distinguishes year counts by their public fields. Both rows are +// legal equality inputs even though comparing their duration requires a +// relative date. The scan, direct index, and indexed Collection read observe +// exact key sets; this test makes no claim about their relative order. +it.each(indexTypes)( + `%s preserves equality indexing for multiple calendar Durations`, + async (_name, IndexType) => { + const rows = [ + { id: `one`, value: Temporal.Duration.from({ years: 1 }) }, + { id: `two`, value: Temporal.Duration.from({ years: 2 }) }, + ] + const probe = Temporal.Duration.from({ years: 2 }) + const expected = new Set( + rows.filter((row) => modelEqual(row.value, probe)).map((row) => row.id), + ) + const index = new IndexType(1, new PropRef([`value`])) + for (const row of rows) index.add(row.id, row) + expect(index.lookup(`eq`, probe)).toEqual(expected) + + const source = createCollection( + mockSyncCollectionOptions({ + id: `calendar-duration-equality-${IndexType.name}`, + getKey: (row: (typeof rows)[number]) => row.id, + initialData: rows, + autoIndex: `off`, + defaultIndexType: IndexType, + }), + ) + try { + await source.stateWhenReady() + const where = eq(new PropRef([`value`]), probe) + const read = () => + new Set(source.currentStateAsChanges({ where })!.map((row) => row.key)) + expect(read()).toEqual(expected) + const publicIndex = source.createIndex((row) => row.value) + const lookup = vi.spyOn(publicIndex, `lookup`) + expect(read()).toEqual(expected) + expect(lookup).toHaveBeenCalled() + } finally { + await source.cleanup() + } + }, +) diff --git a/packages/db/tests/index-update-oracle.property.test.ts b/packages/db/tests/index-update-oracle.property.test.ts index 0cb172df91..bafc931160 100644 --- a/packages/db/tests/index-update-oracle.property.test.ts +++ b/packages/db/tests/index-update-oracle.property.test.ts @@ -1,15 +1,19 @@ import { describe, expect, expectTypeOf, test, vi } from 'vitest' import { fc } from '@fast-check/vitest' +import { Temporal } from 'temporal-polyfill' import { createCollection } from '../src/collection/index.js' import { CollectionInErrorStateError } from '../src/errors.js' import { BasicIndex } from '../src/indexes/basic-index.js' import { BTreeIndex } from '../src/indexes/btree-index.js' +import { eq } from '../src/query/builder/functions.js' import { PropRef } from '../src/query/ir.js' import { DEFAULT_COMPARE_OPTIONS } from '../src/utils.js' -import { indexedKeysSet, orderedEntriesArray, valueMapData } from './utils' +import { mockSyncCollectionOptions } from './utils' import { oraclePropertyOptions, oracleRuns } from './oracle-config' import type { BaseIndex, IndexInterface } from '../src/indexes/base-index.js' import type { CompareOptions } from '../src/query/builder/types.js' +import type { BasicExpression } from '../src/query/ir.js' +import type { StringCollationConfig } from '../src/types.js' /** * An index is a derived multimap from indexed value to source keys. @@ -23,20 +27,35 @@ import type { CompareOptions } from '../src/query/builder/types.js' * A native Map is the sole ownership model. After put, delete, or full build, * the oracle groups that Map by value and freshly sorts the groups. BasicIndex * and BTreeIndex must agree on key count, key membership, equality/range lookup, - * ordered entries, and rebuild behavior. Small duplicate values and signed zero + * full ordered reads, and rebuild behavior. Small duplicate values and signed zero * force collisions in the value groups without copying either index structure. * The comparison runs after every direct index action, after fresh build, and * after each named Collection read. Collection rows and their order are public; * direct index observations are the documented IndexInterface boundary. * The Map retains each key's value because a later same-key put or delete can * distinguish states with the same value multiplicities but different owners. + * An absent lower range bound includes the minimum even when its unused + * `fromInclusive` flag is false. With an explicit exclusive lower bound, + * the value at that bound is excluded. + * Exact equality membership also stays distinct from comparator position: + * replacing a value with one in a different exact-equality bucket at the + * same comparator position must move its exact lookup membership. This law + * concerns returned index results, not bucket identity or maintenance work. + * A build is one of those direct actions even when a source value is + * `undefined`: default nulls-first order must hold after build just as it + * does after incremental add. The nullish lane below exercises both paths. + * A separate mixed number/string lane checks key conservation only. A full + * ordered read and an indexed public snapshot must contain each live source + * key once, even when unlike values have no declared relative sort order. + * A separate two-value callback lane checks legacy scalar input kinds for + * immutable Date, Temporal.Instant, and byte values across both index types. */ type IndexValue = number type IndexConstructor = new ( id: number, - expression: PropRef, + expression: BasicExpression, name?: string, options?: { compareFn?: (left: unknown, right: unknown) => number @@ -91,7 +110,10 @@ const arbitraryAction: fc.Arbitrary = fc.oneof( * updates, delete supplies retirement, and build supplies arbitrary replacement * sets. Removing any one action loses that transition. Removing the narrow * key/value overlap loses collisions; removing -0 loses signed-zero equality. - * Range probes straddle -2, 0, and 2, including their exact boundaries. + * Range probes straddle -2, 0, and 2, including their exact boundaries and + * upper-only reads with an unused exclusive-lower flag and paired reads with + * an explicit exclusive lower bound. Both run after every action, including + * changes to the minimum and an empty build. * Duplicate keys passed directly to build are excluded: the source Map first * resolves them, since duplicate raw build entries have no contract here. */ @@ -99,10 +121,10 @@ const arbitraryAction: fc.Arbitrary = fc.oneof( const probeValues: Array = [-3, -2, -1, -0, 0, 1, 2, 3, 99] const rangeBoundaries: Array = [-2, 0, 2] -function groupKeysByValue( - rows: Map, -): Map> { - const groups = new Map>() +function groupKeysByValue( + rows: ReadonlyMap, +): Map> { + const groups = new Map>() for (const [key, value] of rows) { const keys = groups.get(value) if (keys) { @@ -119,21 +141,22 @@ function expectIndexMatchesModel( rows: Map, ): void { const groups = groupKeysByValue(rows) + const orderedKeys = [...groups] + .sort(([left], [right]) => left - right) + .flatMap(([, keys]) => [...keys].sort()) expect(index.keyCount).toBe(rows.size) - expect(indexedKeysSet(index)).toEqual(new Set(rows.keys())) - expect(valueMapData(index)).toEqual(groups) - // The index may retain either zero sign as a bucket representative. Its - // comparator and the source Map place both signs in the same value group. - expect( - orderedEntriesArray(index).map(([value, keys]) => [ - value === 0 ? 0 : value, - keys, - ]), - ).toEqual([...groups].sort(([left], [right]) => left - right)) + // Complete returned arrays preserve key multiplicity and comparator order + // without requiring a particular bucket or ordered-storage representation. + expect(index.takeFromStart(rows.size + 1)).toEqual(orderedKeys) + expect(index.takeReversedFromEnd(rows.size + 1)).toEqual( + [...orderedKeys].reverse(), + ) for (const value of probeValues) { - expect(index.lookup(`eq`, value)).toEqual(groups.get(value) ?? new Set()) + const expected = groups.get(value) ?? new Set() + expect(index.equalityLookup(value)).toEqual(expected) + expect(index.lookup(`eq`, value)).toEqual(expected) } for (const boundary of rangeBoundaries) { @@ -146,6 +169,22 @@ function expectIndexMatchesModel( expect(index.rangeQuery({ from: boundary })).toEqual(keysAtOrAbove) expect(index.rangeQuery({ to: boundary })).toEqual(keysAtOrBelow) + // The model has no lower predicate here. An inclusivity flag cannot + // exclude the minimum when the caller supplied no lower bound. + expect(index.rangeQuery({ to: boundary, fromInclusive: false })).toEqual( + keysAtOrBelow, + ) + // Supplying the lower bound changes the model predicate. Keys at that + // bound are excluded, while later keys through the upper bound remain. + expect( + index.rangeQuery({ from: boundary, to: 2, fromInclusive: false }), + ).toEqual( + new Set( + [...rows] + .filter(([, value]) => value > boundary && value <= 2) + .map(([key]) => key), + ), + ) expect(index.rangeQueryReversed({ from: boundary })).toEqual(keysAtOrBelow) expect(index.rangeQueryReversed({ to: boundary })).toEqual(keysAtOrAbove) } @@ -309,6 +348,30 @@ describe.each(indexTypes)(`%s update properties`, (_indexName, IndexType) => { expect(index.canOptimizeRangeFor(100)).toBe(false) }) + test.each([ + [`duplicate key`, `takeFromStart`, [`a`, `a`]], + [`wrong reverse order`, `takeReversedFromEnd`, [`a`, `b`]], + ] as const)(`full traversal rejects a %s`, (_name, method, wrongKeys) => { + const index = new IndexType(1, new PropRef([`value`])) + index.add(`a`, { value: 1 }) + index.add(`b`, { value: 2 }) + const wrongOutput = vi.spyOn(index, method).mockReturnValue([...wrongKeys]) + try { + expect(() => + expectIndexMatchesModel( + index, + new Map([ + [`a`, 1], + [`b`, 2], + ]), + ), + ).toThrowError(/expected/) + expect(wrongOutput).toHaveBeenCalled() + } finally { + wrongOutput.mockRestore() + } + }) + test(`inclusive lower bound rejects an exclusive-boundary result`, () => { const index = new IndexType(1, new PropRef([`value`])) index.add(`boundary`, { value: -2 }) @@ -327,6 +390,61 @@ describe.each(indexTypes)(`%s update properties`, (_indexName, IndexType) => { } }) + test(`upper-only range rejects a dropped minimum when the lower flag is exclusive`, () => { + const index = new IndexType(1, new PropRef([`value`])) + index.add(`minimum`, { value: -2 }) + const rangeQuery = index.rangeQuery.bind(index) + const wrongOutput = vi + .spyOn(index, `rangeQuery`) + .mockImplementation((options) => { + const result = rangeQuery(options) + if (!(`from` in options) && options.fromInclusive === false) { + result.delete(`minimum`) + } + return result + }) + try { + expect(() => + expectIndexMatchesModel(index, new Map([[`minimum`, -2]])), + ).toThrowError(/expected/) + } finally { + wrongOutput.mockRestore() + } + }) + + test(`explicit exclusive lower bound rejects an included boundary value`, () => { + const index = new IndexType(1, new PropRef([`value`])) + index.add(`boundary`, { value: -2 }) + index.add(`later`, { value: 0 }) + const rangeQuery = index.rangeQuery.bind(index) + const wrongOutput = vi + .spyOn(index, `rangeQuery`) + .mockImplementation((options) => { + const result = rangeQuery(options) + if ( + options.from === -2 && + options.to === 2 && + options.fromInclusive === false + ) { + result.add(`boundary`) + } + return result + }) + try { + expect(() => + expectIndexMatchesModel( + index, + new Map([ + [`boundary`, -2], + [`later`, 0], + ]), + ), + ).toThrowError(/expected/) + } finally { + wrongOutput.mockRestore() + } + }) + test(`accepts indexed values rather than row keys through the index interface`, () => { const index: IndexInterface = new IndexType( 1, @@ -389,6 +507,813 @@ describe.each(indexTypes)(`%s update properties`, (_indexName, IndexType) => { }) }) +/** + * Direct index equality keeps `undefined` and `null` in distinct exact-value + * buckets, even though the default ordering comparator places both at one + * position. An `IN` lookup returns the union of the requested exact-value + * memberships, including every owner of a repeated value and no unrequested + * value. The established direct-index tests exercise these methods. This law + * does not describe the live-query WHERE evaluator, which uses three-valued + * nullish equality. + * + * A source Map predicts membership without index normalization or comparator + * classification. Fresh singleton histories preserve the old update cuts; + * coexisting values expose a wrong union of nullish buckets. Fixed histories + * also retain two undefined owners beside strings, a nonempty state without + * undefined, and both mixed and undefined-only `IN` lists. A fixed build + * history starts with `undefined` and 1 in both source orders, then continues + * with an add; it distinguishes full rebuild from incremental insertion. + * The generated tail + * adds nearby put/delete/build histories over three keys and five values, with both + * orders of a mixed `IN` list, while a fixed four-key history keeps a third + * defined distractor. An absent source key is added, while an owned key is + * updated. A direct update of an absent key is outside this grammar. Each + * nullish value is needed to distinguish the buckets. Defined values exercise + * mixed-list selection and moves across the nullish boundary. At least two + * keys are needed to expose surviving peers. + * After each direct action returns, exact and `IN` methods, their `lookup` + * dispatches, unfiltered ordered reads, selected forward ranges, and key count must agree with + * the Map. The default comparator puts null and undefined at one ordered + * position before the positive numbers in this bounded domain, although + * exact lookup keeps them apart. Endpoint reads start at an end; an explicit + * cursor excludes its entire comparator position. Range bounds constrain + * positions only when supplied, even when the supplied value is undefined. + * The source Map is sorted anew for ordered reads and filtered for ranges; + * neither expectation uses the production comparator or traversal. Fixed + * histories keep three undefined owners, a remove-one-of-two cut, a mixed + * four-row range, and explicit versus omitted lower bounds. The generated + * tail varies owners and values at each returned-action cut. Build replaces + * source ownership; duplicate generated keys are resolved by the Map before + * invoking production, so a raw duplicate-entry build is excluded. Removing + * build loses the native-sort path; removing the + * nullish values loses the boundary distinction, one owner loses repeated + * bucket traversal, and one defined value loses cursor continuation. Values + * outside the positive 1..3 domain, mixed-type numeric/string order, custom options, + * reverse ranges, all bound combinations, filtered takes, bucket object + * identity, and instrumentation are outside this ordered and range claim. + */ +type NullishIndexValue = number | null | undefined +type NullishIndexAction = + | { type: `put`; key: string; value: NullishIndexValue } + | { type: `delete`; key: string } + | { type: `build`; entries: Array<[string, NullishIndexValue]> } + +const nullishProbeValues: ReadonlyArray = [ + undefined, + null, + 1, + 2, + 3, +] +const nullishInProbes: ReadonlyArray> = [ + [], + [undefined], + [null], + [undefined, 1], + [1, undefined], + [undefined, null], + [3], + [1, 2], + [1, 1], +] +const arbitraryNullishAction: fc.Arbitrary = fc.oneof( + fc.record({ + type: fc.constant(`put` as const), + key: fc.constantFrom(`a`, `b`, `c`), + value: fc.constantFrom(...nullishProbeValues), + }), + fc.record({ + type: fc.constant(`delete` as const), + key: fc.constantFrom(`a`, `b`, `c`), + }), + fc.record({ + type: fc.constant(`build` as const), + entries: fc.array( + fc.tuple( + fc.constantFrom(`a`, `b`, `c`), + fc.constantFrom(...nullishProbeValues), + ), + { maxLength: 3 }, + ), + }), +) + +function expectExactMembership( + index: BaseIndex, + rows: ReadonlyMap, + exactProbes: ReadonlyArray, + inProbes: ReadonlyArray>, +): void { + // Map key identity distinguishes null from undefined without consulting the + // index's normalized keys or its ordering comparator. IN membership follows + // from source-value membership in the operand list, including an empty list. + const groups = groupKeysByValue(rows) + expect(index.keyCount).toBe(rows.size) + const traversedKeys = index.takeFromStart(rows.size + 1) + expect(traversedKeys).toHaveLength(rows.size) + expect(new Set(traversedKeys)).toEqual(new Set(rows.keys())) + for (const value of exactProbes) { + const expected = groups.get(value) ?? new Set() + expect(index.equalityLookup(value)).toEqual(expected) + expect(index.lookup(`eq`, value)).toEqual(expected) + } + for (const values of inProbes) { + const expected = new Set( + [...rows] + .filter(([, value]) => values.includes(value)) + .map(([key]) => key), + ) + expect(index.inArrayLookup([...values])).toEqual(expected) + expect(index.lookup(`in`, [...values])).toEqual(expected) + } +} + +function expectNullishMembership( + index: BaseIndex, + rows: ReadonlyMap, +): void { + expectExactMembership(index, rows, nullishProbeValues, nullishInProbes) + expectNullishOrderedReads(index, rows) +} + +type ModelBound = Readonly<{ + value: NullishIndexValue + inclusive: boolean +}> +type ModelRange = Readonly<{ + lower?: ModelBound + upper?: ModelBound + unusedLowerInclusive?: false +}> + +const nullishRangeProbes: ReadonlyArray = [ + {}, + { upper: { value: 2, inclusive: true } }, + { upper: { value: 2, inclusive: true }, unusedLowerInclusive: false }, + { + lower: { value: undefined, inclusive: true }, + upper: { value: 2, inclusive: true }, + }, + { + lower: { value: undefined, inclusive: false }, + upper: { value: 2, inclusive: true }, + }, + { + lower: { value: undefined, inclusive: true }, + upper: { value: undefined, inclusive: true }, + }, + { + lower: { value: null, inclusive: true }, + upper: { value: null, inclusive: true }, + }, + { lower: { value: null, inclusive: false } }, + { upper: { value: undefined, inclusive: false } }, + { + lower: { value: 1, inclusive: true }, + upper: { value: 2, inclusive: false }, + }, +] + +// "Position" is a model label, not a stored index bucket. In the declared +// domain 1..3, zero names the shared nullish comparator position. Source key +// ownership remains in the Map even when two values share that position. +function nullishPosition(value: NullishIndexValue): number { + return value === null || value === undefined ? 0 : value +} + +function expectNullishOrderedReads( + index: BaseIndex, + rows: ReadonlyMap, +): void { + const ordered = [...rows] + .sort( + ([leftKey, leftValue], [rightKey, rightValue]) => + nullishPosition(leftValue) - nullishPosition(rightValue) || + (leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0), + ) + .map(([key, value]) => ({ key, value })) + + // The public arrays preserve both ordering and multiplicity. A set-only + // check would miss a repeated undefined owner or a skipped peer. + for (const n of [1, 3, 10]) { + expect(index.takeFromStart(n)).toEqual( + ordered.slice(0, n).map(({ key }) => key), + ) + expect(index.takeReversedFromEnd(n)).toEqual( + [...ordered] + .reverse() + .slice(0, n) + .map(({ key }) => key), + ) + for (const cursor of nullishProbeValues) { + const cursorPosition = nullishPosition(cursor) + expect(index.take(n, cursor)).toEqual( + ordered + .filter(({ value }) => nullishPosition(value) > cursorPosition) + .slice(0, n) + .map(({ key }) => key), + ) + expect(index.takeReversed(n, cursor)).toEqual( + [...ordered] + .reverse() + .filter(({ value }) => nullishPosition(value) < cursorPosition) + .slice(0, n) + .map(({ key }) => key), + ) + } + } + + // Model bounds are separate objects, so an absent lower bound is not + // conflated with a present bound whose value is literally undefined. + for (const probe of nullishRangeProbes) { + const options = { + ...(probe.lower && { + from: probe.lower.value, + fromInclusive: probe.lower.inclusive, + }), + ...(probe.upper && { + to: probe.upper.value, + toInclusive: probe.upper.inclusive, + }), + ...(probe.unusedLowerInclusive === false && { fromInclusive: false }), + } + const expected = new Set( + ordered + .filter(({ value }) => { + const position = nullishPosition(value) + const lower = probe.lower + const upper = probe.upper + return ( + (lower === undefined || + (lower.inclusive + ? position >= nullishPosition(lower.value) + : position > nullishPosition(lower.value))) && + (upper === undefined || + (upper.inclusive + ? position <= nullishPosition(upper.value) + : position < nullishPosition(upper.value))) + ) + }) + .map(({ key }) => key), + ) + expect(index.rangeQuery(options)).toEqual(expected) + } +} + +function applyNullishAction( + index: BaseIndex, + rows: Map, + action: NullishIndexAction, +): void { + // The driver maps source ownership to the real add, update, and remove API. + if (action.type === `put`) { + if (rows.has(action.key)) { + index.update( + action.key, + { value: rows.get(action.key) }, + { value: action.value }, + ) + } else { + index.add(action.key, { value: action.value }) + } + rows.set(action.key, action.value) + } else if (action.type === `build`) { + // Resolve duplicate generated source keys before calling build; their raw + // duplicate-entry policy is outside this law. Build replaces the Map. + const replacement = new Map(action.entries) + index.build([...replacement].map(([key, value]) => [key, { value }])) + rows.clear() + for (const [key, value] of replacement) rows.set(key, value) + } else if (rows.has(action.key)) { + index.remove(action.key, { value: rows.get(action.key) }) + rows.delete(action.key) + } + expectNullishMembership(index, rows) +} + +describe.each(indexTypes)( + `%s nullish direct index refinement`, + (_indexName, IndexType) => { + test.each([ + [`undefined to null`, undefined, null], + [`null to undefined`, null, undefined], + [`defined to undefined`, 1, undefined], + [`undefined to defined`, undefined, 1], + [`undefined to undefined`, undefined, undefined], + ] as const)(`keeps membership after %s`, (_label, before, after) => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { type: `put`, key: `a`, value: before }) + applyNullishAction(index, rows, { type: `put`, key: `a`, value: after }) + }) + + test(`keeps coexisting null and undefined buckets distinct`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [key, value] of [ + [`null1`, null], + [`undef1`, undefined], + [`num1`, 1], + ] as const) { + applyNullishAction(index, rows, { type: `put`, key, value }) + } + }) + + test(`finds every undefined owner beside distinct string values`, () => { + const index = new IndexType(1, new PropRef([`status`])) + const rows = new Map() + const exactProbes = [undefined, `active`, `inactive`] + const inProbes = [[undefined], [undefined, `active`]] + for (const [key, value] of [ + [`a`, `active`], + [`b`, undefined], + [`c`, `inactive`], + [`d`, undefined], + ] as const) { + index.add(key, { status: value }) + rows.set(key, value) + expectExactMembership(index, rows, exactProbes, inProbes) + } + }) + + test(`returns no undefined key from a nonempty all-defined index`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { type: `put`, key: `a`, value: 1 }) + applyNullishAction(index, rows, { type: `put`, key: `b`, value: 2 }) + }) + + test(`unites undefined and defined IN members without distractors`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [key, value] of [ + [`undef`, undefined], + [`one`, 1], + [`two`, 2], + [`three`, 3], + ] as const) { + applyNullishAction(index, rows, { type: `put`, key, value }) + } + }) + + test(`returns every undefined owner from a singleton IN list`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [key, value] of [ + [`undef1`, undefined], + [`undef2`, undefined], + [`one`, 1], + ] as const) { + applyNullishAction(index, rows, { type: `put`, key, value }) + } + }) + + test(`distinguishes endpoint reads from explicit undefined cursors`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [key, value] of [ + [`a`, undefined], + [`b`, 1], + [`c`, 2], + ] as const) { + applyNullishAction(index, rows, { type: `put`, key, value }) + } + }) + + test(`traverses every undefined owner before continuing to numbers`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [key, value] of [ + [`undef1`, undefined], + [`undef2`, undefined], + [`undef3`, undefined], + [`one`, 1], + [`two`, 2], + ] as const) { + applyNullishAction(index, rows, { type: `put`, key, value }) + } + }) + + test(`retains the second undefined owner after removing the first`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { + type: `put`, + key: `a`, + value: undefined, + }) + applyNullishAction(index, rows, { + type: `put`, + key: `b`, + value: undefined, + }) + applyNullishAction(index, rows, { type: `delete`, key: `a` }) + }) + + test(`distinguishes explicit and omitted undefined lower bounds`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [key, value] of [ + [`undef`, undefined], + [`one`, 1], + [`two`, 2], + [`three`, 3], + ] as const) { + applyNullishAction(index, rows, { type: `put`, key, value }) + } + }) + + test(`includes both undefined owners within double undefined bounds`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [key, value] of [ + [`a`, undefined], + [`b`, undefined], + [`c`, 1], + ] as const) { + applyNullishAction(index, rows, { type: `put`, key, value }) + } + }) + + test(`build preserves nulls-first ordered reads and ranges`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { + type: `build`, + entries: [ + [`undefined`, undefined], + [`one`, 1], + ], + }) + // A second build changes insertion order; the model still predicts the + // same returned key order and complete upper-bound results. + applyNullishAction(index, rows, { + type: `build`, + entries: [ + [`one`, 1], + [`undefined`, undefined], + ], + }) + applyNullishAction(index, rows, { + type: `put`, + key: `two`, + value: 2, + }) + applyNullishAction(index, rows, { + type: `build`, + entries: [ + [`undef2`, undefined], + [`null`, null], + [`undef1`, undefined], + [`one`, 1], + ], + }) + }) + + test.each([20261012, undefined])( + `refines nullish membership histories (seed %s)`, + async (seed) => { + await fc.assert( + fc.asyncProperty( + fc.array(arbitraryNullishAction, { minLength: 1, maxLength: 40 }), + async (actions) => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + expectNullishMembership(index, rows) + for (const action of actions) { + applyNullishAction(index, rows, action) + } + }, + ), + seed === undefined + ? oraclePropertyOptions(100, `index-update.nullish-membership`) + : { seed, numRuns: oracleRuns(100) }, + ) + }, + ) + + test(`rejects a skipped undefined-to-null update`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { + type: `put`, + key: `a`, + value: undefined, + }) + const skipped = vi.spyOn(index, `update`).mockImplementation(() => {}) + try { + expect(() => + applyNullishAction(index, rows, { + type: `put`, + key: `a`, + value: null, + }), + ).toThrowError(/expected/) + expect(skipped).toHaveBeenCalledOnce() + } finally { + skipped.mockRestore() + } + }) + + test(`rejects lookup dispatch that unites nullish buckets`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { + type: `put`, + key: `null1`, + value: null, + }) + applyNullishAction(index, rows, { + type: `put`, + key: `undef1`, + value: undefined, + }) + const lookup = index.lookup.bind(index) + const united = vi + .spyOn(index, `lookup`) + .mockImplementation((op, value) => + op === `eq` && value === undefined + ? new Set([`null1`, `undef1`]) + : lookup(op, value), + ) + try { + expect(() => expectNullishMembership(index, rows)).toThrowError( + /expected/, + ) + expect(united).toHaveBeenCalledWith(`eq`, undefined) + } finally { + united.mockRestore() + } + }) + + test(`rejects an exact lookup that loses one undefined owner`, () => { + const index = new IndexType(1, new PropRef([`status`])) + const rows = new Map([ + [`a`, `active`], + [`b`, undefined], + [`c`, `inactive`], + [`d`, undefined], + ]) + for (const [key, status] of rows) index.add(key, { status }) + const lookup = index.equalityLookup.bind(index) + const truncated = vi + .spyOn(index, `equalityLookup`) + .mockImplementation((value) => + value === undefined ? new Set([`b`]) : lookup(value), + ) + try { + expect(() => + expectExactMembership(index, rows, [undefined], [[undefined]]), + ).toThrowError(/expected/) + expect(truncated).toHaveBeenCalledWith(undefined) + } finally { + truncated.mockRestore() + } + }) + + test(`rejects an IN lookup that drops an undefined operand`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { + type: `put`, + key: `undef`, + value: undefined, + }) + applyNullishAction(index, rows, { type: `put`, key: `one`, value: 1 }) + const lookup = index.inArrayLookup.bind(index) + const dropped = vi + .spyOn(index, `inArrayLookup`) + .mockImplementation((values) => + values.length === 2 && values[0] === 1 && values[1] === undefined + ? new Set([`one`]) + : lookup(values), + ) + try { + expect(() => expectNullishMembership(index, rows)).toThrowError( + /expected/, + ) + expect(dropped).toHaveBeenCalledWith([1, undefined]) + } finally { + dropped.mockRestore() + } + }) + + test(`rejects an IN lookup that loses a same-value owner`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { + type: `put`, + key: `undef1`, + value: undefined, + }) + applyNullishAction(index, rows, { + type: `put`, + key: `undef2`, + value: undefined, + }) + const lookup = index.inArrayLookup.bind(index) + const truncated = vi + .spyOn(index, `inArrayLookup`) + .mockImplementation((values) => + values.length === 1 && values[0] === undefined + ? new Set([`undef1`]) + : lookup(values), + ) + try { + expect(() => expectNullishMembership(index, rows)).toThrowError( + /expected/, + ) + expect(truncated).toHaveBeenCalledWith([undefined]) + } finally { + truncated.mockRestore() + } + }) + + test(`rejects IN dispatch that returns only its first value group`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + applyNullishAction(index, rows, { + type: `put`, + key: `undef`, + value: undefined, + }) + applyNullishAction(index, rows, { type: `put`, key: `one`, value: 1 }) + const lookup = index.lookup.bind(index) + const firstOnly = vi + .spyOn(index, `lookup`) + .mockImplementation((op, value) => + op === `in` && + Array.isArray(value) && + value.length === 2 && + value[0] === undefined && + value[1] === 1 + ? new Set([`undef`]) + : lookup(op, value), + ) + try { + expect(() => expectNullishMembership(index, rows)).toThrowError( + /expected/, + ) + expect(firstOnly).toHaveBeenCalledWith(`in`, [undefined, 1]) + } finally { + firstOnly.mockRestore() + } + }) + + test(`rejects an endpoint read that drops the undefined owner`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map([ + [`a`, undefined], + [`b`, 1], + [`c`, 2], + ]) + for (const [key, value] of rows) index.add(key, { value }) + const dropped = vi + .spyOn(index, `takeFromStart`) + .mockImplementation(() => [`b`, `c`]) + try { + expect(() => expectNullishOrderedReads(index, rows)).toThrowError( + /expected/, + ) + expect(dropped).toHaveBeenCalled() + } finally { + dropped.mockRestore() + } + }) + + test(`rejects a cursor that repeats the last undefined owner`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map([ + [`undef1`, undefined], + [`undef2`, undefined], + [`undef3`, undefined], + [`one`, 1], + [`two`, 2], + ]) + for (const [key, value] of rows) index.add(key, { value }) + const take = index.take.bind(index) + const repeated = vi + .spyOn(index, `take`) + .mockImplementation((n, from, filter) => + n === 10 && from === undefined + ? [`undef3`, `one`, `two`] + : take(n, from, filter), + ) + try { + expect(() => expectNullishOrderedReads(index, rows)).toThrowError( + /expected/, + ) + expect(repeated).toHaveBeenCalledWith(10, undefined) + } finally { + repeated.mockRestore() + } + }) + + test(`rejects a forward range that drops an explicit undefined lower bound`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map([ + [`undef`, undefined], + [`one`, 1], + [`two`, 2], + [`three`, 3], + ]) + for (const [key, value] of rows) index.add(key, { value }) + const rangeQuery = index.rangeQuery.bind(index) + const dropped = vi + .spyOn(index, `rangeQuery`) + .mockImplementation((options) => + `from` in options && + options.from === undefined && + options.to === 2 && + options.fromInclusive === true + ? new Set([`one`, `two`]) + : rangeQuery(options), + ) + try { + expect(() => expectNullishOrderedReads(index, rows)).toThrowError( + /expected/, + ) + expect(dropped).toHaveBeenCalledWith({ + from: undefined, + fromInclusive: true, + to: 2, + toInclusive: true, + }) + } finally { + dropped.mockRestore() + } + }) + + test(`rejects a double undefined range that loses one owner`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map([ + [`a`, undefined], + [`b`, undefined], + [`c`, 1], + ]) + for (const [key, value] of rows) index.add(key, { value }) + const rangeQuery = index.rangeQuery.bind(index) + const truncated = vi + .spyOn(index, `rangeQuery`) + .mockImplementation((options) => + `from` in options && + options.from === undefined && + `to` in options && + options.to === undefined + ? new Set([`a`]) + : rangeQuery(options), + ) + try { + expect(() => expectNullishOrderedReads(index, rows)).toThrowError( + /expected/, + ) + expect(truncated).toHaveBeenCalledWith({ + from: undefined, + fromInclusive: true, + to: undefined, + toInclusive: true, + }) + } finally { + truncated.mockRestore() + } + }) + + test(`rejects a null bound that drops its undefined peer`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map([ + [`a`, undefined], + [`b`, null], + [`c`, 1], + ]) + for (const [key, value] of rows) index.add(key, { value }) + const rangeQuery = index.rangeQuery.bind(index) + const exactOnly = vi + .spyOn(index, `rangeQuery`) + .mockImplementation((options) => + `from` in options && + options.from === null && + `to` in options && + options.to === null + ? new Set([`b`]) + : rangeQuery(options), + ) + try { + expect(() => expectNullishOrderedReads(index, rows)).toThrowError( + /expected/, + ) + expect(exactOnly).toHaveBeenCalledWith({ + from: null, + fromInclusive: true, + to: null, + toInclusive: true, + }) + } finally { + exactOnly.mockRestore() + } + }) + }, +) + /** * A custom index comparator must return a number that is not NaN. * @@ -408,7 +1333,10 @@ describe.each(indexTypes)(`%s update properties`, (_indexName, IndexType) => { * operation. The probe inserts or reads the non-numeric value `ann`. An * empty prefix is included: its first write compares nothing, so it must * succeed, and the next comparing operation must throw. Comparator - * transitivity is outside this owner. + * transitivity is outside this owner. A removal need not compare a surviving + * value with its neighbors: the former representative-promotion example + * prescribed one storage layout, while the rejected-removal witness above + * retains the atomicity law for comparisons that actually occur. */ describe.each(indexTypes)( `%s invalid comparator results`, @@ -546,25 +1474,19 @@ describe.each(indexTypes)( expect(index.lookup(`eq`, `x`)).toEqual(new Set([`x`])) }) - // `A` and `a` share one comparator position. Removing the representative - // `A` promotes `a`, whose comparison with `z` is the only invalid pair. - test(`a rejected representative replacement leaves the index unchanged`, () => { - const { index, step } = createRecordedIndex((left, right) => { - if ([left, right].includes(`a`) && [left, right].includes(`z`)) - return NaN - const l = String(left).toLowerCase() - const r = String(right).toLowerCase() - return l === r ? 0 : l < r ? -1 : 1 - }) - expect(step(() => index.add(`upper`, { value: `A` }))).toBe(false) - expect(step(() => index.add(`lower`, { value: `a` }))).toBe(false) - expect(step(() => index.add(`zed`, { value: `z` }))).toBe(false) - // Equality lookups and key count read the index without comparing. - if (step(() => index.remove(`upper`, { value: `A` }))) { - expect(index.keyCount).toBe(3) - expect(index.lookup(`eq`, `A`)).toEqual(new Set([`upper`])) - expect(index.lookup(`eq`, `a`)).toEqual(new Set([`lower`])) - expect(index.lookup(`eq`, `z`)).toEqual(new Set([`zed`])) + test(`a rejected remove preserves other exact values`, () => { + const { index, step } = createRecordedIndex((left, right) => + left === `x` && right === `x` + ? NaN + : String(left).localeCompare(String(right)), + ) + for (const value of [`a`, `x`, `z`]) { + expect(step(() => index.add(value, { value }))).toBe(false) + } + expect(step(() => index.remove(`x`, { value: `x` }))).toBe(true) + expect(index.keyCount).toBe(3) + for (const value of [`a`, `x`, `z`]) { + expect(index.lookup(`eq`, value)).toEqual(new Set([value])) } }) @@ -819,14 +1741,114 @@ describe.each(indexTypes)(`%s comparator groups`, (_indexName, IndexType) => { expectCustomGroups(index, rows, identities) }) + test(`moves exact string membership across a comparator-equal update`, () => { + // The input facts put A/a in one order group and B/b in another. The + // comparator implements that order, but the expected exact memberships + // come from the source Map's unchanged strings, not from the comparator. + const group = { A: 0, a: 0, B: 1, b: 1 } as const + type Value = keyof typeof group + const rows = new Map() + const index = new IndexType(1, new PropRef([`value`]), undefined, { + compareFn: (left, right) => + (left as string) + .toLowerCase() + .localeCompare((right as string).toLowerCase()), + }) + const expectMatchesSource = () => { + expect(index.keyCount).toBe(rows.size) + for (const value of Object.keys(group) as Array) { + const exactKeys = new Set( + [...rows] + .filter(([, current]) => current === value) + .map(([key]) => key), + ) + expect(index.equalityLookup(value)).toEqual(exactKeys) + expect(index.lookup(`eq`, value)).toEqual(exactKeys) + expect(index.rangeQuery({ from: value, to: value })).toEqual( + new Set( + [...rows] + .filter(([, current]) => group[current] === group[value]) + .map(([key]) => key), + ), + ) + } + } + + // Check after each direct index action. First update a singleton, as in + // the legacy witness; then add peers in both comparator groups. A shortcut + // limited to singleton indexes must fail before the peers can mask it. + index.add(`changed`, { value: `A` }) + rows.set(`changed`, `A`) + expectMatchesSource() + + const skipEqualUpdate = vi.spyOn(index, `update`).mockImplementation(() => { + if (index.keyCount === 1) return + }) + try { + index.update(`changed`, { value: `A` }, { value: `a` }) + rows.set(`changed`, `a`) + expect(() => expectMatchesSource()).toThrowError(/expected/) + } finally { + skipEqualUpdate.mockRestore() + rows.set(`changed`, `A`) + } + + for (const [key, oldValue, newValue] of [ + [`changed`, `A`, `a`], + [`changed`, `a`, `A`], + ] as Array<[string, Value, Value]>) { + index.update(key, { value: oldValue }, { value: newValue }) + rows.set(key, newValue) + expectMatchesSource() + } + + index.add(`anchor`, { value: `a` }) + rows.set(`anchor`, `a`) + expectMatchesSource() + index.add(`other`, { value: `B` }) + rows.set(`other`, `B`) + expectMatchesSource() + + // These histories additionally retain another exact value in the same + // order group and then move a key across groups. + for (const [key, oldValue, newValue] of [ + [`changed`, `A`, `a`], + [`other`, `B`, `b`], + [`other`, `b`, `a`], + ] as Array<[string, Value, Value]>) { + index.update(key, { value: oldValue }, { value: newValue }) + rows.set(key, newValue) + expectMatchesSource() + } + + // EqualityLookup and the public lookup dispatch are separate methods. + // A dispatch that substitutes the comparator group for exact equality + // would pass an equalityLookup-only check after the same-group update. + const lookup = index.lookup.bind(index) + const wrongLookup = vi + .spyOn(index, `lookup`) + .mockImplementation((operation, value) => + operation === `eq` + ? index.rangeQuery({ from: value, to: value }) + : lookup(operation, value), + ) + try { + expect(() => expectMatchesSource()).toThrowError(/expected/) + } finally { + wrongLookup.mockRestore() + } + }) + // Group grammar: 2..20 generated IDs in 0..4, with a forced duplicate first // group and a distinct group 5. The same bounded input is run in both index // types. The exact-identity lane gives every row a fresh array, then retires // and reuses handles. The custom-comparator lane additionally reverses // insertion. Without the duplicate, representative retirement is absent; // without the distinct group, group order is unobservable; without reversed - // insertion, insertion order can masquerade as lexical key order. The pinned - // [0, 1] case reconstructs all three distinctions. Invalid comparator + // insertion, insertion order can masquerade as lexical key order. Replacing + // one value with a fresh value in the same group checks exact membership + // independently of comparator position; a later update crosses groups. + // The pinned [0, 1] case reconstructs these distinctions. Invalid comparator // results and inconsistent comparator order are outside these two grammars. test.each([20260930, undefined])( `preserves exact equality while ordered traversal retains every row (seed %s)`, @@ -885,15 +1907,6 @@ describe.each(indexTypes)(`%s comparator groups`, (_indexName, IndexType) => { expect( subject.takeReversedFromEnd(currentRows.length + 1), ).toEqual(reversed) - for (const [representative, keys] of orderedEntriesArray( - subject, - )) { - expect( - currentRows.some( - (row) => row.value === representative && keys.has(row.key), - ), - ).toBe(true) - } for (const row of currentRows) { expect(subject.equalityLookup(row.value)).toEqual( new Set([row.key]), @@ -1023,6 +2036,23 @@ describe.each(indexTypes)(`%s comparator groups`, (_indexName, IndexType) => { expectCustomGroups(index, rows, identities) const previous = rows[0]! + const equalGroup = Object.freeze({ + key: previous.key, + groupId: previous.groupId, + value: { + groupId: previous.groupId, + position: previous.value.position, + }, + }) + identities.push(equalGroup.value) + index.update( + previous.key, + { value: previous.value }, + { value: equalGroup.value }, + ) + rows[0] = equalGroup + expectCustomGroups(index, rows, identities) + const changed = Object.freeze({ key: previous.key, groupId: 99, @@ -1031,16 +2061,20 @@ describe.each(indexTypes)(`%s comparator groups`, (_indexName, IndexType) => { identities.push(changed.value) index.update( previous.key, - { value: previous.value }, + { value: equalGroup.value }, { value: changed.value }, ) rows[0] = changed expectCustomGroups(index, rows, identities) - for (const retired of [removed, previous]) { + for (const [ordinal, retired] of [ + removed, + previous, + equalGroup, + ].entries()) { const reused = Object.freeze({ ...retired, - key: `reused-${retired.key}`, + key: `reused-${ordinal}-${retired.key}`, }) index.add(reused.key, { value: reused.value }) rows.push(reused) @@ -1150,7 +2184,7 @@ const collationCases = [ ] as const function createCollationCollection( - collation: (typeof collationCases)[number][`collation`], + collation: StringCollationConfig, autoIndex: `off` | `eager`, defaultIndexType?: typeof BasicIndex | typeof BTreeIndex, labels: ReadonlyArray = [`item10`, `item2`], @@ -1164,8 +2198,8 @@ function createCollationCollection( sync: { sync: ({ begin, write, commit, markReady }) => { begin() - for (const label of labels) { - write({ type: `insert`, value: { id: label, label } }) + for (const [ordinal, label] of labels.entries()) { + write({ type: `insert`, value: { id: String(ordinal), label } }) } commit() markReady() @@ -1271,3 +2305,502 @@ test(`ordered scan resolves inherited collation outside each comparison`, async expect(collection.indexes.size).toBe(0) }) }) + +/** + * Public ordered reads compare the user's string values. An index escapes a + * leading internal prefix for exact-key storage; that storage encoding must + * not change the Collection's declared string order. The en-US locale and a + * custom callback both place the prefixed value first. The scan and the + * bounded indexed read must return that same first row. + */ +const prefixedLabel = `\u0000tanstack-db:a` +const plainLabel = `tanstack-db:r` +const prefixedStringCases = [ + { + name: `locale`, + collation: { stringSort: `locale`, locale: `en-US` }, + }, + { + name: `custom collation`, + collation: { + stringSort: `custom`, + compare: (left: string, right: string) => + left.localeCompare(right, `en-US`), + }, + }, +] as const satisfies ReadonlyArray<{ + name: string + collation: StringCollationConfig +}> + +describe.each([ + [`BasicIndex`, BasicIndex], + [`BTreeIndex`, BTreeIndex], +] as const)(`%s prefixed-string public order`, (_name, IndexType) => { + test.each(prefixedStringCases)( + `$name scan and bounded index return the same first row`, + async ({ collation }) => { + const labels = [plainLabel, prefixedLabel] + expect(prefixedLabel.localeCompare(plainLabel, `en-US`)).toBeLessThan(0) + const orderBy = [ + { + expression: new PropRef([`label`]), + compareOptions: { + direction: `asc` as const, + nulls: `first` as const, + }, + }, + ] + + const scan = createCollationCollection( + collation, + `off`, + undefined, + labels, + ) + await withCollectionCleanup(scan, async () => { + await scan.stateWhenReady() + const changes = scan.currentStateAsChanges({ orderBy, limit: 1 }) + expect(changes?.map(({ value }) => value.label)).toEqual([ + prefixedLabel, + ]) + expect(scan.indexes.size, `scan path reached`).toBe(0) + }) + + const indexed = createCollationCollection( + collation, + `eager`, + IndexType, + labels, + ) + await withCollectionCleanup(indexed, async () => { + await indexed.stateWhenReady() + const changes = indexed.currentStateAsChanges({ + orderBy, + limit: 1, + optimizedOnly: true, + }) + expect(changes?.map(({ value }) => value.label)).toEqual([ + prefixedLabel, + ]) + expect(indexed.indexes.size, `index path reached`).toBe(1) + }) + }, + ) +}) + +/** + * Mixed primitive values can share a comparator position without sharing an + * exact value. The source Map owns each key once. This model intentionally + * predicts membership and multiplicity, not the relative order of a number + * and a string. Its fixed grammar adds 2, "2", and "10", then removes each + * possible owner in a fresh history. A second history deletes, updates across + * value kinds, then re-adds an earlier key. A third history adds two ordered + * Temporal values beside an ordinary string that resembles an internal key, + * then deletes one Temporal owner. After each returned action, both full index + * traversals must contain exactly the live source keys. + * + * In the public lane a ready Collection receives the same rows and deletes + * the number. A manual index serves a bounded `currentStateAsChanges` read + * with room for one more row than the source contains. The returned change + * messages must contain each live key once. `optimizedOnly` and the take spy + * establish that this observation came through the index. No mixed-kind + * `orderBy` sequence, custom callback, or modified built-in is claimed. + */ +const mixedRows: ReadonlyArray<{ id: string; value: number | string }> = [ + { id: `a`, value: 2 }, + { id: `b`, value: `2` }, + { id: `c`, value: `10` }, +] + +function expectConservedIndexKeys( + index: BaseIndex, + rows: ReadonlyMap, +): void { + const expected = new Set(rows.keys()) + expect(index.keyCount).toBe(expected.size) + for (const keys of [ + index.takeFromStart(rows.size + 1), + index.takeReversedFromEnd(rows.size + 1), + ]) { + expect(keys).toHaveLength(expected.size) + expect(new Set(keys)).toEqual(expected) + } +} + +describe.each(indexTypes)( + `%s mixed-value key conservation`, + (_indexName, IndexType) => { + test.each(mixedRows)( + `keeps every surviving key once after deleting $id`, + (removed) => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const row of mixedRows) { + index.add(row.id, row) + rows.set(row.id, row.value) + expectConservedIndexKeys(index, rows) + } + index.remove(removed.id, removed) + rows.delete(removed.id) + expectConservedIndexKeys(index, rows) + }, + ) + + test(`keeps one copy of each key after a mixed-value delete, update, and add`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + const add = (id: string, value: number | string) => { + index.add(id, { id, value }) + rows.set(id, value) + expectConservedIndexKeys(index, rows) + } + + add(`a`, 2) + add(`b`, `2`) + add(`d`, `10`) + index.remove(`a`, { id: `a`, value: 2 }) + rows.delete(`a`) + expectConservedIndexKeys(index, rows) + index.update(`b`, { id: `b`, value: `2` }, { id: `b`, value: 2 }) + rows.set(`b`, 2) + expectConservedIndexKeys(index, rows) + add(`a`, `2`) + }) + + test.each([ + { + name: `ZonedDateTime`, + first: + Temporal.Instant.fromEpochNanoseconds(9n).toZonedDateTimeISO(`UTC`), + second: + Temporal.Instant.fromEpochNanoseconds(10n).toZonedDateTimeISO(`UTC`), + text: `tanstack-db:temporal:Temporal.ZonedDateTime:5:fake`, + }, + { + name: `Duration`, + first: Temporal.Duration.from({ nanoseconds: 9 }), + second: Temporal.Duration.from({ nanoseconds: 10 }), + text: `tanstack-db:temporal:Temporal.Duration:time:5:fake`, + }, + ])( + `keeps $name and ordinary-string owners after deletion`, + ({ first, second, text }) => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + for (const [id, value] of [ + [`a`, first], + [`b`, second], + [`c`, text], + ] as const) { + index.add(id, { id, value }) + rows.set(id, value) + expectConservedIndexKeys(index, rows) + } + index.remove(`b`, { id: `b`, value: second }) + rows.delete(`b`) + expectConservedIndexKeys(index, rows) + }, + ) + }, +) + +describe.each([ + [`BasicIndex`, BasicIndex], + [`BTreeIndex`, BTreeIndex], +] as const)(`%s mixed-value public snapshot`, (_indexName, IndexType) => { + test(`returns each surviving key once after a source deletion`, async () => { + const options = mockSyncCollectionOptions<{ + id: string + value: number | string + }>({ + id: `mixed-value-public-snapshot-${IndexType.name}`, + getKey: (row) => row.id, + initialData: mixedRows.map((row) => ({ ...row })), + autoIndex: `off`, + }) + const collection = createCollection(options) + await withCollectionCleanup(collection, async () => { + await collection.stateWhenReady() + const index = collection.createIndex((row) => row.value, { + indexType: IndexType, + }) + const take = vi.spyOn(index, `takeFromStart`) + const rows = new Map(mixedRows.map((row) => [row.id, row.value])) + + options.utils.begin() + options.utils.write({ type: `delete`, value: mixedRows[0]! }) + options.utils.commit() + rows.delete(mixedRows[0]!.id) + + take.mockClear() + + const changes = collection.currentStateAsChanges({ + orderBy: [ + { + expression: new PropRef([`value`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + limit: rows.size + 1, + optimizedOnly: true, + }) + expect(take).toHaveBeenCalled() + expect(changes).toBeDefined() + const keys = changes!.map((change) => change.key) + expect(keys).toHaveLength(rows.size) + expect(new Set(keys)).toEqual(new Set(rows.keys())) + }) + }) +}) + +/** + * Binary equality is by byte content, so a fresh byte array can remove an + * earlier equal value. The source Map predicts exact membership after removal, + * re-addition, and replacement while another byte value remains. Complete + * index reads must contain each live key once; byte ordering is unspecified. + */ +describe.each(indexTypes)(`%s binary-value ownership`, (_name, IndexType) => { + test(`does not retain an ordered entry after equal-content removal`, () => { + const index = new IndexType(1, new PropRef([`value`])) + const rows = new Map() + const bytes = (value = 1) => new Uint8Array([value]) + const check = () => { + const expected = new Set(rows.keys()) + expect(index.keyCount).toBe(rows.size) + expect(index.equalityLookup(bytes())).toEqual( + new Set( + [...rows].filter(([, value]) => value[0] === 1).map(([key]) => key), + ), + ) + for (const read of [ + index.takeFromStart(rows.size + 1), + index.takeReversedFromEnd(rows.size + 1), + ]) { + expect(read).toHaveLength(rows.size) + expect(new Set(read)).toEqual(expected) + } + } + + index.add(`a`, { value: bytes() }) + rows.set(`a`, bytes()) + check() + index.remove(`a`, { value: bytes() }) + rows.delete(`a`) + check() + index.add(`b`, { value: bytes() }) + rows.set(`b`, bytes()) + check() + + const oldBytes = bytes() + index.add(`c`, { value: bytes(2) }) + rows.set(`c`, bytes(2)) + index.add(`d`, { value: oldBytes }) + rows.set(`d`, oldBytes) + check() + index.remove(`b`, { value: bytes() }) + rows.delete(`b`) + check() + index.update(`d`, { value: oldBytes }, { value: bytes() }) + rows.set(`d`, bytes()) + check() + }) + + test.each([2, 8193])( + `compares every byte for exact membership (%i bytes)`, + (length) => { + const index = new IndexType(1, new PropRef([`value`])) + const left = new Uint8Array(length) + left[0] = 1 + const right = left.slice() + right[length - 1] = 1 + index.add(`left`, { value: left }) + index.add(`right`, { value: right }) + expect(index.equalityLookup(left.slice())).toEqual(new Set([`left`])) + expect(index.equalityLookup(right.slice())).toEqual(new Set([`right`])) + for (const read of [ + index.takeFromStart(3), + index.takeReversedFromEnd(3), + ]) { + expect(read).toHaveLength(2) + expect(new Set(read)).toEqual(new Set([`left`, `right`])) + } + }, + ) +}) + +/** + * PlainMonthDay has exact equality but no intrinsic order. The two-row source + * Map predicts one key for a copy of February 1, both before and after index + * creation. The index may choose any storage order; a public equality read + * must not fail merely because both values are present. + */ +describe.each(indexTypes)( + `%s unordered Temporal equality`, + (_name, IndexType) => { + test(`accepts distinct PlainMonthDay values for exact lookup`, async () => { + const rows = [ + { id: `jan`, value: Temporal.PlainMonthDay.from(`--01-01`) }, + { id: `feb`, value: Temporal.PlainMonthDay.from(`--02-01`) }, + ] + const probe = Temporal.PlainMonthDay.from(`--02-01`) + const expected = new Set( + rows + .filter( + (row) => + row.value.monthCode === probe.monthCode && + row.value.day === probe.day, + ) + .map((row) => row.id), + ) + const index = new IndexType(1, new PropRef([`value`])) + for (const row of rows) index.add(row.id, row) + expect(index.lookup(`eq`, probe)).toEqual(expected) + + const collection = createCollection( + mockSyncCollectionOptions({ + id: `month-day-equality-${IndexType.name}`, + getKey: (row: (typeof rows)[number]) => row.id, + initialData: rows, + autoIndex: `off`, + defaultIndexType: IndexType, + }), + ) + await withCollectionCleanup(collection, async () => { + await collection.stateWhenReady() + const where = eq(new PropRef([`value`]), probe) + const read = () => + collection + .currentStateAsChanges({ where })! + .map((row) => row.key) + .sort() + expect(read()).toEqual([...expected].sort()) + const publicIndex = collection.createIndex((row) => row.value) + const lookup = vi.spyOn(publicIndex, `lookup`) + expect(read()).toEqual([...expected].sort()) + expect(lookup).toHaveBeenCalled() + }) + }) + }, +) + +/** + * A bounded read should not make more comparator calls as retired values + * accumulate. A fixed history retains one row while adding and removing 16, + * 64, or 256 earlier values. A user comparator records calls made by the + * public takeFromStart method after each history. This witnesses comparator + * work only: a traversal over tombstones that avoids the comparator could + * pass, and retained memory is measured separately. + */ +describe.each(indexTypes)(`%s retired-value read work`, (_name, IndexType) => { + test(`takeFromStart comparator work does not grow with retired values`, () => { + const comparisons = [16, 64, 256].map((retiredCount) => { + let calls = 0 + const index = new IndexType(1, new PropRef([`value`]), undefined, { + compareFn: (left: unknown, right: unknown) => { + calls++ + return (left as number) - (right as number) + }, + }) + index.add(`live`, { value: 1000 }) + for (let value = 0; value < retiredCount; value++) { + const key = String(value) + index.add(key, { value }) + index.remove(key, { value }) + } + expect(index.keyCount).toBe(1) + calls = 0 + expect(index.takeFromStart(1)).toEqual([`live`]) + return calls + }) + + expect(comparisons[2]).toBeLessThanOrEqual(comparisons[0]! + 4) + }) +}) + +/** + * Distinct exact values can share one comparator position. An ordered first + * row still requires sorting that group's keys, but it need not recompare + * every value in the group. The fixed 16/64/256 sizes distinguish a grouped + * read from one tree search per tied exact value. This bounds comparator work + * at the direct read checkpoint, not elapsed time or retained memory. + */ +test(`BTreeIndex bounded reads do not recompare every live tie`, () => { + const comparisons = [16, 64, 256].map((count) => { + let calls = 0 + const index = new BTreeIndex(1, new PropRef([`value`]), undefined, { + compareFn: () => { + calls++ + return 0 + }, + }) + for (let value = 0; value < count; value++) { + index.add(String(value), { value }) + } + calls = 0 + expect(index.takeFromStart(1)).toEqual([`0`]) + return calls + }) + expect(comparisons[2]).toBeLessThanOrEqual(comparisons[0]! + 4) +}) + +/** + * Preserve a bounded legacy callback input: older indexes supplied a Date's + * timestamp and a string for Temporal and binary values. For two immutable + * values of each named kind, a direct add comparison supplies that same scalar + * kind in both index types. The model uses two values with a known order; it + * does not copy or specify the index's string encoding. A callback that accepts + * only the old scalar kind distinguishes a raw-object leak. After both adds, + * full ordered keys and exact owners must match the supplied facts. This does + * not establish arbitrary callbacks, build/update/range paths, or mutation of + * the value objects after admission. + */ +describe.each(indexTypes)( + `%s comparator input compatibility`, + (_name, IndexType) => { + test.each([ + { + name: `Date`, + values: [new Date(1), new Date(2)], + kind: `number`, + }, + { + name: `Temporal.Instant`, + values: [ + Temporal.Instant.from(`2020-01-01T00:00:00Z`), + Temporal.Instant.from(`2021-01-01T00:00:00Z`), + ], + kind: `string`, + }, + { + name: `Uint8Array`, + values: [new Uint8Array([1]), new Uint8Array([2])], + kind: `string`, + }, + ])( + `passes $name to callbacks in its legacy scalar kind`, + ({ values, kind }) => { + let comparisons = 0 + const index = new IndexType(1, new PropRef([`value`]), undefined, { + compareFn: (left, right) => { + comparisons++ + expect(typeof left).toBe(kind) + expect(typeof right).toBe(kind) + if (kind === `number`) return (left as number) - (right as number) + const a = left as string + const b = right as string + return a < b ? -1 : a > b ? 1 : 0 + }, + }) + // Keys oppose value order, so collapsing both inputs to one comparator + // position cannot pass by key-tie ordering alone. + index.add(`a`, { value: values[1] }) + index.add(`z`, { value: values[0] }) + expect(comparisons).toBeGreaterThan(0) + expect(index.takeFromStart(2)).toEqual([`z`, `a`]) + expect(index.equalityLookup(values[0])).toEqual(new Set([`z`])) + expect(index.equalityLookup(values[1])).toEqual(new Set([`a`])) + }, + ) + }, +) diff --git a/packages/db/tests/index-update-short-circuit.test.ts b/packages/db/tests/index-update-short-circuit.test.ts index 216bc5110d..bf22354a6e 100644 --- a/packages/db/tests/index-update-short-circuit.test.ts +++ b/packages/db/tests/index-update-short-circuit.test.ts @@ -78,29 +78,6 @@ describe.each(indexTypes)(`%s update`, (_indexName, IndexType) => { expect(index.lookup(`eq`, 2)).toEqual(new Set([`a`])) }) - it(`does not conflate undefined and null`, () => { - const index = createIndex() - index.add(`a`, { value: undefined }) - - index.update(`a`, { value: undefined }, { value: null }) - - expect(index.lookup(`eq`, undefined)).toEqual(new Set()) - expect(index.lookup(`eq`, null)).toEqual(new Set([`a`])) - }) - - it(`does not use comparator equality to skip an update`, () => { - const index = createIndex({ - compareFn: (a: string, b: string) => - a.toLowerCase().localeCompare(b.toLowerCase()), - }) - index.add(`a`, { value: `A` }) - - index.update(`a`, { value: `A` }, { value: `a` }) - - expect(index.lookup(`eq`, `A`)).toEqual(new Set()) - expect(index.lookup(`eq`, `a`)).toEqual(new Set([`a`])) - }) - it(`preserves the previous error behavior when evaluation fails`, () => { const index = createIndex() index.add(`a`, { value: 1 }) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index cb3ac26996..bafcd87e00 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -61,6 +61,7 @@ const staticOracleProperties = [ `cursor.partial-local-order`, `cursor.repeat-construction`, `index-update.reference-model`, + `index-update.nullish-membership`, `index-update.exact-identity`, `index-update.custom-comparator`, `query-identity.compiled-output`, diff --git a/packages/db/tests/oracle-replay-manifest.ts b/packages/db/tests/oracle-replay-manifest.ts index 37e85d04d9..bea5072439 100644 --- a/packages/db/tests/oracle-replay-manifest.ts +++ b/packages/db/tests/oracle-replay-manifest.ts @@ -32,7 +32,7 @@ const ownerGroups: ReadonlyArray = [ [ `db/tests/index-update-oracle.property.test.ts`, `index-update`, - `reference-model exact-identity custom-comparator`, + `reference-model nullish-membership exact-identity custom-comparator`, ], [ `db/tests/cursor-oracle.property.test.ts`, @@ -266,6 +266,10 @@ const indexReplayTestNames = new Map([ `index-update.reference-model`, `matches a reference model across valid operation sequences \\(seed undefined\\)`, ], + [ + `index-update.nullish-membership`, + `refines nullish membership histories \\(seed undefined\\)`, + ], [ `index-update.exact-identity`, `preserves exact equality while ordered traversal retains every row \\(seed undefined\\)`, diff --git a/packages/db/tests/oracle-replay.test.ts b/packages/db/tests/oracle-replay.test.ts index de172a716b..539419facd 100644 --- a/packages/db/tests/oracle-replay.test.ts +++ b/packages/db/tests/oracle-replay.test.ts @@ -100,6 +100,11 @@ describe(`guarded oracle replay`, () => { `tests/index-update-oracle.property.test.ts`, 2, ], + [ + `index-update.nullish-membership`, + `tests/index-update-oracle.property.test.ts`, + 2, + ], [ `index-update.exact-identity`, `tests/index-update-oracle.property.test.ts`, @@ -428,7 +433,7 @@ describe(`named oracle replay manifest`, () => { const direct = oracleReplayManifest.filter(({ property }) => /^(index-update|pagination\.matrix)\./.test(property), ) - expect(direct).toHaveLength(19) + expect(direct).toHaveLength(20) for (const entry of direct) { expect(entry.status).toBe(`assertion`) expect(entry.testNamePattern).toBeDefined() diff --git a/packages/db/tests/query/compiler/binary-equality-work.test.ts b/packages/db/tests/query/compiler/binary-equality-work.test.ts index ddbbeed670..ce17fdeafb 100644 --- a/packages/db/tests/query/compiler/binary-equality-work.test.ts +++ b/packages/db/tests/query/compiler/binary-equality-work.test.ts @@ -66,7 +66,7 @@ describe(`binary equality work`, () => { ) it.each([`eq`, `in`])( - `compares a MiB using %s without caching mutable bytes`, + `compares a MiB using %s without encoding strings`, (operator) => { const left = new Uint8Array(1024 * 1024).fill(65) const right = left.slice() @@ -78,8 +78,9 @@ describe(`binary equality work`, () => { ) const equal = measureEncoding(() => evaluate({ blob: left })) expect(equal).toEqual({ result: true, bytes: 0 }) - right[right.length - 1] = 66 - const different = measureEncoding(() => evaluate({ blob: left })) + const changedRow = left.slice() + changedRow[changedRow.length - 1] = 66 + const different = measureEncoding(() => evaluate({ blob: changedRow })) expect(different).toEqual({ result: false, bytes: 0 }) }, ) diff --git a/packages/db/tests/query/compiler/in-evaluator-oracle.property.test.ts b/packages/db/tests/query/compiler/in-evaluator-oracle.property.test.ts index f5ea6b05cb..bca8b0ed6e 100644 --- a/packages/db/tests/query/compiler/in-evaluator-oracle.property.test.ts +++ b/packages/db/tests/query/compiler/in-evaluator-oracle.property.test.ts @@ -19,7 +19,8 @@ import { oraclePropertyOptions, oracleRuns } from '../../oracle-config.js' * - A valid Date equals its millisecond timestamp, and another Date with the * same timestamp. * - Two Uint8Arrays (Buffers included) are equal when their bytes are equal. - * - A Temporal value equals a Temporal value of the same kind and string. + * - A Temporal value equals one of the same kind and string, except Duration: + * its ten fields determine equality even when two forms display alike. * - Every other pair compares with `===`, so `-0` equals `0`, `1n` does not * equal `1`, and objects compare by reference. * @@ -73,6 +74,9 @@ const pool: ReadonlyArray = [ Temporal.PlainDate.from(`2024-01-01`), Temporal.PlainDate.from(`2024-01-01`), Temporal.PlainTime.from(`10:00`), + Temporal.Duration.from({ seconds: 1 }), + Temporal.Duration.from({ seconds: 1 }), + Temporal.Duration.from({ milliseconds: 1000 }), shared, { id: 1 }, null, @@ -111,6 +115,19 @@ function asComparable(value: unknown): unknown { return value instanceof Date ? value.getTime() : value } +const durationFields = [ + `years`, + `months`, + `weeks`, + `days`, + `hours`, + `minutes`, + `seconds`, + `milliseconds`, + `microseconds`, + `nanoseconds`, +] as const + function referenceEqual(left: unknown, right: unknown): boolean { if (isNotANumber(left) || isNotANumber(right)) { return isNotANumber(left) && isNotANumber(right) @@ -123,6 +140,13 @@ function referenceEqual(left: unknown, right: unknown): boolean { Array.from(left).every((byte, index) => byte === right[index]) ) } + if (left instanceof Temporal.Duration || right instanceof Temporal.Duration) { + return ( + left instanceof Temporal.Duration && + right instanceof Temporal.Duration && + durationFields.every((field) => left[field] === right[field]) + ) + } if (isTemporalValue(left) || isTemporalValue(right)) { return ( isTemporalValue(left) && @@ -219,6 +243,18 @@ describe(`in evaluator`, () => { [`equal bytes`, Buffer.from([1, 2]), [new Uint8Array([1, 2])], true], [`bytes and their internal key`, PREFIX, [new Uint8Array([])], false], [`a structural copy`, { id: 1 }, [shared], false], + [ + `equal Duration fields`, + Temporal.Duration.from({ seconds: 1 }), + [Temporal.Duration.from({ seconds: 1 })], + true, + ], + [ + `equivalent Duration forms with different fields`, + Temporal.Duration.from({ seconds: 1 }), + [Temporal.Duration.from({ milliseconds: 1000 })], + false, + ], [`an empty list`, `a`, [], false], ] as const)(`pins %s`, (_name, value, list, expected) => { const testCase = { value, list: [...list], listFromRow: false } @@ -240,4 +276,18 @@ describe(`in evaluator`, () => { ), ).toThrow(/in mismatch/) }) + + it(`rejects display-text equality for equivalent Duration forms`, () => { + const seconds = Temporal.Duration.from({ seconds: 1 }) + const milliseconds = Temporal.Duration.from({ milliseconds: 1000 }) + const textual: Evaluate = ({ value, list }) => + list.some((entry) => String(entry) === String(value)) + expect(() => + check(textual, { + value: seconds, + list: [milliseconds], + listFromRow: false, + }), + ).toThrow(/in mismatch/) + }) }) diff --git a/packages/db/tests/query/custom-index-order-selection-oracle.test.ts b/packages/db/tests/query/custom-index-order-selection-oracle.test.ts new file mode 100644 index 0000000000..622e421864 --- /dev/null +++ b/packages/db/tests/query/custom-index-order-selection-oracle.test.ts @@ -0,0 +1,118 @@ +/** + * An index may choose how to store rows, but it cannot change a query's + * declared order. For a bounded `orderBy(rank)` query, the public window is + * the ascending ranks from the source rows, regardless of a manual index's + * callback. The live-query guide defines `orderBy` and `limit`; the live-query + * architecture treats index selection as physical planning. + * + * The Collection API also exposes an ordered, limited direct snapshot through + * `currentStateAsChanges`. The model sorts a plain array by numeric rank and + * slices it. Its three distinct ranks avoid any promise about callback ordering + * within an equality class. The legal history starts one ready source, then + * reads a direct snapshot and compiles one limited live query. Eager and + * prepopulated on-demand sources each run with a scan and a reverse-order + * BTree index. Three distinct ranks and a two-row window can expose a reversed + * prefix whose membership and order differ from the query's prefix. The public row + * IDs must equal the model at the direct-read and post-preload checkpoints. + * Taking the reverse index's first two entries as the query window would + * expose ranks 30 and 20 instead of 10 and 20. The oracle checks the public + * result and allows any physical plan that produces it. + * The on-demand adapter confirms a finite request without adding rows because + * all three are already installed. This witness does not cover provider paging + * from an incomplete source or later source changes. + */ +import { expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { createLiveQueryCollection } from '../../src/query/live-query-collection.js' +import { PropRef } from '../../src/query/ir.js' +import { mockSyncCollectionOptions, withOracleCleanup } from '../utils.js' + +type Row = { id: number; rank: number } + +const rows: ReadonlyArray = [ + { id: 30, rank: 30 }, + { id: 20, rank: 20 }, + { id: 10, rank: 10 }, +] + +// The query's rank order comes from its declaration, not index traversal. +const expectedIds = [...rows] + .sort((a, b) => a.rank - b.rank) + .slice(0, 2) + .map((row) => row.id) + +function makeSource( + mode: `eager` | `on-demand`, + path: `scan` | `reverse manual index`, +) { + const id = `custom-index-order-selection-${mode}-${path}` + if (mode === `eager`) { + return createCollection( + mockSyncCollectionOptions({ + id, + getKey: (row) => row.id, + initialData: [...rows], + autoIndex: `off`, + }), + ) + } + return createCollection({ + id, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + commit() + markReady() + return { loadSubset: () => true, unloadSubset: () => {} } + }, + }, + }) +} + +it.each( + ([`eager`, `on-demand`] as const).flatMap((mode) => + ([`scan`, `reverse manual index`] as const).map((path) => ({ mode, path })), + ), +)( + `keeps bounded public rank order with $mode $path`, + async ({ mode, path }) => { + const source = makeSource(mode, path) + let live: ReturnType | undefined + await withOracleCleanup(async () => { + if (mode === `eager`) await source.stateWhenReady() + else expect(source.isReady()).toBe(true) + if (path === `reverse manual index`) { + source.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareFn: (a: number, b: number) => b - a }, + }) + } + + const direct = source.currentStateAsChanges({ + orderBy: [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + limit: 2, + })! + expect(direct.map((change) => change.value.id)).toEqual(expectedIds) + + live = createLiveQueryCollection((q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(2), + ) + await live.preload() + expect([...live.values()].map((row) => row.id)).toEqual(expectedIds) + }, [() => live?.cleanup(), () => source.cleanup()]) + }, +) diff --git a/packages/db/tests/query/mixed-index-order-oracle.test.ts b/packages/db/tests/query/mixed-index-order-oracle.test.ts new file mode 100644 index 0000000000..6d3ba8489e --- /dev/null +++ b/packages/db/tests/query/mixed-index-order-oracle.test.ts @@ -0,0 +1,449 @@ +/** + * An index is a physical route for an `orderBy` query. It cannot choose a + * different public prefix from a full-source scan of the same query, either + * on the first read or after a source value changes kind. This follows the + * `orderBy`/`limit` contract in docs/guides/live-queries.md; it does not define + * a new global precedence among unlike JavaScript values. + * + * The reference is a separate ready Collection with auto-indexing disabled. + * It holds the same keyed source rows as the indexed Collection and executes + * the same public query. Its full scan is independent of index storage and the + * index-only comparator being judged. The finite histories cross number, + * bigint, boolean, string, ordinary-object, and array kinds, then update, + * insert, or delete rows to enter and leave mixed domains. Ascending and + * descending reads also exercise the reverse index adapter. Values are not + * mutated in place. Every value set has a distinct first and second row, so + * the law does not depend on an unspecified tie order. These witnesses do not + * prescribe order for every mixed-kind combination or provider paging. + * + * After preload and after both source copies commit each step, direct ordered snapshots + * and both live-query Collections must return the same exact ordered IDs. + * The scan and indexed sources publish identical sync transactions. Running + * both BasicIndex and BTreeIndex proves the optimizer cannot silently change + * the query result. The original type-name index order loses at the exact + * public-prefix comparison; a same-type step checks that the repair still + * permits indexed traversal when its order agrees with the query. + */ +import { expect, it, vi } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { createLiveQueryCollection } from '../../src/query/live-query-collection.js' +import { PropRef } from '../../src/query/ir.js' +import { evaluateReferenceExpression } from '../reference-expression-oracle.js' +import { mockSyncCollectionOptions, withOracleCleanup } from '../utils.js' +import type { Collection } from '../../src/collection/index.js' +import type { + LoadSubsetOptions, + SyncConfig, + UtilsRecord, +} from '../../src/types.js' +import type { BasicExpression } from '../../src/query/ir.js' +import type { BaseIndex } from '../../src/indexes/base-index.js' + +type Value = number | bigint | boolean | string | { x: number } | Array +type Row = { id: string; value: Value } +type Step = + { type: `insert` | `update`; row: Row } | { type: `delete`; id: string } +type Scenario = { + name: string + initial: Array + steps: Array + indexedReads: Array +} + +const histories: Array = [ + { + name: `number and bigint`, + initial: [ + { id: `five`, value: 5n }, + { id: `ten`, value: 10n }, + { id: `three`, value: 3 }, + ], + steps: [ + { type: `update`, row: { id: `three`, value: 12n } }, + { type: `insert`, row: { id: `minus`, value: -4 } }, + { type: `delete`, id: `minus` }, + ], + indexedReads: [false, true, false, true], + }, + { + name: `number index gains and loses a bigint`, + initial: [ + { id: `three`, value: 3 }, + { id: `five`, value: 5 }, + ], + steps: [ + { type: `insert`, row: { id: `minus`, value: -1n } }, + { type: `update`, row: { id: `minus`, value: 7 } }, + { type: `update`, row: { id: `minus`, value: -2n } }, + ], + indexedReads: [true, false, true, false], + }, + { + name: `boolean and number`, + initial: [ + { id: `false`, value: false }, + { id: `five`, value: 5 }, + { id: `minus`, value: -1 }, + ], + steps: [ + { type: `update`, row: { id: `false`, value: 7 } }, + { type: `insert`, row: { id: `true`, value: true } }, + { type: `delete`, id: `true` }, + ], + indexedReads: [false, true, false, true], + }, + { + name: `object and string`, + initial: [ + { id: `object`, value: { x: 1 } }, + { id: `b`, value: `b` }, + { id: `a`, value: `a` }, + ], + steps: [ + { type: `update`, row: { id: `object`, value: `c` } }, + { type: `insert`, row: { id: `other`, value: { x: 2 } } }, + { type: `delete`, id: `other` }, + ], + indexedReads: [false, true, false, true], + }, + { + name: `array and ordinary object`, + initial: [ + { id: `array`, value: [1] }, + { id: `object`, value: { x: 1 } }, + ], + steps: [ + { type: `update`, row: { id: `array`, value: { x: 2 } } }, + { type: `insert`, row: { id: `array-again`, value: [2] } }, + { type: `delete`, id: `array-again` }, + ], + indexedReads: [false, true, false, true], + }, +] + +function orderedSnapshot( + source: Pick< + Collection, + `currentStateAsChanges` + >, + direction: `asc` | `desc`, +): Array { + return source + .currentStateAsChanges({ + orderBy: [ + { + expression: new PropRef([`value`]), + compareOptions: { direction, nulls: `first` }, + }, + ], + limit: 2, + })! + .map((change) => change.value.id) +} + +function applyStep( + source: ReturnType>, + rows: Map, + step: Step, +): void { + source.utils.begin() + if (step.type === `delete`) { + source.utils.write({ type: `delete`, value: rows.get(step.id)! }) + rows.delete(step.id) + } else { + source.utils.write({ type: step.type, value: step.row }) + rows.set(step.row.id, step.row) + } + source.utils.commit() +} + +for (const scenario of histories) { + for (const IndexType of [BasicIndex, BTreeIndex]) { + for (const direction of [`asc`, `desc`] as const) { + it(`${IndexType.name} preserves ${scenario.name} ${direction} public order across kind changes`, async () => { + const scanOptions = mockSyncCollectionOptions({ + id: `mixed-order-scan-${scenario.name}-${IndexType.name}`, + getKey: (row) => row.id, + initialData: scenario.initial, + autoIndex: `off`, + }) + const indexedOptions = mockSyncCollectionOptions({ + id: `mixed-order-indexed-${scenario.name}-${IndexType.name}`, + getKey: (row) => row.id, + initialData: scenario.initial, + autoIndex: `eager`, + defaultIndexType: IndexType, + }) + const scan = createCollection(scanOptions) + const indexed = createCollection(indexedOptions) + const scanRows = new Map(scenario.initial.map((row) => [row.id, row])) + const indexedRows = new Map(scanRows) + let scanLive: ReturnType | undefined + let indexedLive: + ReturnType | undefined + await withOracleCleanup(async () => { + await Promise.all([scan.stateWhenReady(), indexed.stateWhenReady()]) + // Keep one ascending physical index so descending reads must use + // ReverseIndex rather than auto-creating a descending index. + indexed.createIndex((row) => row.value, { indexType: IndexType }) + const check = (checkpoint: string, expectIndexRead: boolean) => { + const expected = orderedSnapshot(scan, direction) + const indexes = [...indexed.indexes.values()] + expect(indexes, `${checkpoint}: index exists`).toHaveLength(1) + const index = indexes[0] as BaseIndex + expect(index, `${checkpoint}: index type`).toBeInstanceOf(IndexType) + expect(index.matchesDirection(`asc`)).toBe(true) + const take = vi.spyOn( + index, + direction === `asc` ? `takeFromStart` : `takeReversedFromEnd`, + ) + try { + expect( + orderedSnapshot(indexed, direction), + `${checkpoint}: direct prefix`, + ).toEqual(expected) + if (expectIndexRead) + expect(take, `${checkpoint}: indexed route`).toHaveBeenCalled() + else + expect( + take, + `${checkpoint}: full-scan route`, + ).not.toHaveBeenCalled() + } finally { + take.mockRestore() + } + expect( + [...scanLive!.values()].map((row) => row.id), + `${checkpoint}: scan live prefix`, + ).toEqual(expected) + expect( + [...indexedLive!.values()].map((row) => row.id), + `${checkpoint}: indexed live prefix`, + ).toEqual(expected) + } + // Compile both live queries while the initial source domain is mixed. + scanLive = createLiveQueryCollection((q) => + q + .from({ row: scan }) + .orderBy(({ row }) => row.value, direction) + .limit(2), + ) + indexedLive = createLiveQueryCollection((q) => + q + .from({ row: indexed }) + .orderBy(({ row }) => row.value, direction) + .limit(2), + ) + await Promise.all([scanLive.preload(), indexedLive.preload()]) + check(`initial domain`, scenario.indexedReads[0]!) + for (const [position, step] of scenario.steps.entries()) { + applyStep(scanOptions, scanRows, step) + applyStep(indexedOptions, indexedRows, step) + check( + `step ${position + 1}: ${step.type}`, + scenario.indexedReads[position + 1]!, + ) + } + }, [ + () => indexedLive?.cleanup(), + () => scanLive?.cleanup(), + () => indexed.cleanup(), + () => scan.cleanup(), + ]) + }) + } + } +} + +/** + * The eager histories above do not cross provider demand. This receiving + * history begins with a homogeneous numeric index and a finite on-demand + * provider. A later bigint insert makes that retained index ineligible for + * a new ordered plan. The independent provider filters requested predicates, + * applies cursors, and returns only the requested page from its own source + * rows. At each settled window, the model sorts those rows by numeric value + * and compares the exact public IDs. The bigint first enters the visible + * prefix, then moves beyond it; a subsequent deletion requires a missing + * provider row to enter the bounded result. A later page reaches a cursor. + * This checks the public result, not whether an existing loader avoids every + * unnecessary index read. + */ +type NumericRow = { id: string; value: number | bigint } + +// The shared request reference intentionally accepts only like-typed numeric +// operands. This local extension gives the provider its legal number/bigint +// cursor semantics using JavaScript's direct relational order. +function evaluateNumericRequest( + expression: BasicExpression, + row: NumericRow, +): unknown { + if (expression.type !== `func`) + return evaluateReferenceExpression(expression, row) + if (expression.name === `and` || expression.name === `or`) { + const values = expression.args.map((arg) => + evaluateNumericRequest(arg, row), + ) + return expression.name === `and` + ? values.every(Boolean) + : values.some(Boolean) + } + if ([`gt`, `gte`, `lt`, `lte`].includes(expression.name)) { + const [left, right] = expression.args.map((arg) => + evaluateNumericRequest(arg, row), + ) + if ( + (typeof left === `number` || typeof left === `bigint`) && + (typeof right === `number` || typeof right === `bigint`) + ) { + const sign = left < right ? -1 : left > right ? 1 : 0 + switch (expression.name) { + case `gt`: + return sign > 0 + case `gte`: + return sign >= 0 + case `lt`: + return sign < 0 + case `lte`: + return sign <= 0 + } + } + } + return evaluateReferenceExpression(expression, row) +} + +it(`preserves a retained on-demand window after its index domain becomes mixed`, async () => { + const initial: Array = [ + { id: `three`, value: 3 }, + { id: `five`, value: 5 }, + { id: `ten`, value: 10 }, + { id: `twenty`, value: 20 }, + { id: `thirty`, value: 30 }, + ] + const truth = new Map(initial.map((row) => [row.id, row])) + const installed = new Map(initial.slice(0, 2).map((row) => [row.id, row])) + const requests: Array = [] + let sync!: Parameters[`sync`]>[0] + const compare = (a: NumericRow, b: NumericRow) => + a.value < b.value ? -1 : a.value > b.value ? 1 : 0 + const expected = (offset = 0, limit = 2) => + [...truth.values()] + .sort(compare) + .slice(offset, offset + limit) + .map((row) => row.id) + + const source = createCollection({ + id: `mixed-index-order-on-demand`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + sync: { + sync: (operations) => { + sync = operations + sync.begin() + for (const row of installed.values()) { + sync.write({ type: `insert`, value: row }) + } + sync.commit() + sync.markReady() + return { + loadSubset: async (options) => { + requests.push(options) + const matched = [...truth.values()] + .filter( + (row) => + (!options.where || + evaluateNumericRequest(options.where, row) === true) && + (!options.cursor || + evaluateNumericRequest(options.cursor.whereFrom, row) === + true), + ) + .sort(compare) + const offset = options.cursor ? 0 : (options.offset ?? 0) + const page = matched.slice( + offset, + options.limit === undefined ? undefined : offset + options.limit, + ) + sync.begin() + for (const row of page) { + if (installed.get(row.id) === row) continue + sync.write({ + type: installed.has(row.id) ? `update` : `insert`, + value: row, + }) + installed.set(row.id, row) + } + await sync.commit() + }, + unloadSubset: () => {}, + } + }, + }, + }) + let live: ReturnType | undefined + let take: { mockRestore: () => void } | undefined + await withOracleCleanup(async () => { + await source.stateWhenReady() + const index = source.createIndex((row) => row.value, { + indexType: BTreeIndex, + }) + expect(index.supportsOrderOptimization).toBe(true) + take = vi.spyOn(index, `takeFromStart`) + live = createLiveQueryCollection((q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.value) + .limit(2), + ) + await live.preload() + expect([...live.values()].map((row) => row.id)).toEqual(expected()) + expect(take).toHaveBeenCalled() + + const mixed: NumericRow = { id: `four`, value: 4n } + truth.set(mixed.id, mixed) + sync.begin() + sync.write({ type: `insert`, value: mixed }) + installed.set(mixed.id, mixed) + await sync.commit() + expect(index.supportsOrderOptimization).toBe(false) + await live.utils.setWindow({ offset: 0, limit: 2 }) + expect(expected()).toEqual([`three`, `four`]) + expect([...live.values()].map((row) => row.id)).toEqual(expected()) + + const moved: NumericRow = { id: `four`, value: 15n } + truth.set(moved.id, moved) + sync.begin() + sync.write({ type: `update`, value: moved }) + installed.set(moved.id, moved) + await sync.commit() + await live.utils.setWindow({ offset: 0, limit: 2 }) + expect(expected()).toEqual([`three`, `five`]) + expect([...live.values()].map((row) => row.id)).toEqual(expected()) + expect(installed.has(`ten`)).toBe(false) + + const requestsBeforeDeletion = requests.length + truth.delete(`three`) + sync.begin() + sync.write({ type: `delete`, value: initial[0]! }) + installed.delete(`three`) + await sync.commit() + await live.utils.setWindow({ offset: 0, limit: 2 }) + expect(expected()).toEqual([`five`, `ten`]) + expect([...live.values()].map((row) => row.id)).toEqual(expected()) + expect(installed.has(`ten`)).toBe(true) + expect( + requests + .slice(requestsBeforeDeletion) + .some((request) => request.orderBy && request.limit === 2), + ).toBe(true) + + const requestsBeforePage = requests.length + await live.utils.setWindow({ offset: 2, limit: 2 }) + expect([...live.values()].map((row) => row.id)).toEqual(expected(2, 2)) + expect( + requests.slice(requestsBeforePage).some((request) => request.cursor), + ).toBe(true) + }, [() => take?.mockRestore(), () => live?.cleanup(), () => source.cleanup()]) +}) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index a293c5a53b..53cabd4c0a 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -1,4 +1,3 @@ -import { runInNewContext } from 'node:vm' import { describe, expect, it, vi } from 'vitest' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe' import { eq, gt } from '../../src/query/builder/functions' @@ -383,49 +382,6 @@ describe(`DeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) - it(`does not substitute comparison payloads with custom instance methods`, () => { - const date = new Date(2) - const bytes = new Uint8Array([1, 2, 3]) - Object.defineProperty(date, `getTime`, { value: () => 1 }) - Object.defineProperty(bytes, `slice`, { value: () => bytes }) - const where = new Func(`and`, [ - eq(ref(`date`), val(date)), - eq(ref(`bytes`), val(bytes)), - ]) - const request = captureRequest({ where }) - const rows = [ - { date, bytes }, - { date: new Date(2), bytes: new Uint8Array([1, 2, 3]) }, - ] - expect(request.where).toBe(where) - expect(rows.map(compileSingleRowExpression(request.where!))).toEqual( - rows.map(compileSingleRowExpression(where)), - ) - }) - - describe.each([`Date`, `Uint8Array`] as const)( - `request transport preserves %s predicate matches`, - (type) => { - it.each([`local`, `foreign`] as const)(`in the %s realm`, (realm) => { - const local = type === `Date` ? new Date(2) : new Uint8Array([1, 2]) - const foreign: unknown = runInNewContext( - type === `Date` ? `new Date(2)` : `new Uint8Array([1, 2])`, - ) - const value = realm === `local` ? local : foreign - for (const where of [ - eq(ref(`value`), val(value)), - new Func(`in`, [ref(`value`), val([value])]), - ]) { - const request = captureRequest({ where }) - const matches = compileSingleRowExpression(request.where!) - expect( - [foreign, local].map((item) => matches({ value: item })), - ).toEqual(realm === `foreign` ? [true, false] : [false, true]) - } - }) - }, - ) - it.each([`coalesce`, `caseWhen`] as const)( `preserves membership results through %s`, (wrapper) => { diff --git a/packages/db/tests/temporal-index-order-oracle.test.ts b/packages/db/tests/temporal-index-order-oracle.test.ts new file mode 100644 index 0000000000..3e041f0022 --- /dev/null +++ b/packages/db/tests/temporal-index-order-oracle.test.ts @@ -0,0 +1,644 @@ +import { Temporal } from 'temporal-polyfill' +import { describe, expect, it, vi } from 'vitest' +import { BasicIndex } from '../src/indexes/basic-index' +import { BTreeIndex } from '../src/indexes/btree-index' +import { createCollection } from '../src/collection/index' +import { createLiveQueryCollection } from '../src/query/live-query-collection' +import { PropRef } from '../src/query/ir' +import { mockSyncCollectionOptions, withOracleCleanup } from './utils' +import type { CompareOptions } from '../src/query/builder/types' + +/** + * # Does an index preserve the public order of valid ZonedDateTimes? + * + * Changelog #1519 promises instant order for ZonedDateTime `orderBy`. The + * supplied instants below put `early` first, while their display strings put + * `late` first. A plain two-row model therefore predicts `early, late` for an + * ascending direct index read and `early` for a one-row public query. + * + * The legal fixed grammar crosses BasicIndex and BTreeIndex with incremental + * adds and full builds, positive and negative epoch width, string collations, + * direction, and equal-instant ties, then compares exact ordered keys after + * the action. A ready source exercises scan and indexed live-query paths at + * preload; an on-demand Collection checks the first published ordered row. + * Index spies prove the named indexed reads. The PlainDate lane checks one + * extended-year calendar boundary; the exception lane requires `TypeError` + * for its named mixed or unordered Temporal pairs and claims no ordered + * result for them. Custom index callbacks, mutated built-ins, and other + * mixed Temporal pairs remain outside this owner. + */ +type Row = { id: string; value: Temporal.ZonedDateTime } +const rows: ReadonlyArray = [ + { + id: `early`, + value: Temporal.ZonedDateTime.from(`2024-01-01T14:00+14:00[+14:00]`), + }, + { + id: `late`, + value: Temporal.ZonedDateTime.from(`2024-01-01T01:00+00:00[UTC]`), + }, +] +const [early, late] = rows +const expectedOrder = [`early`, `late`] +const widthBoundary: ReadonlyArray = [ + { + id: `early`, + value: + Temporal.Instant.fromEpochNanoseconds(9n).toZonedDateTimeISO(`+14:00`), + }, + { + id: `late`, + value: Temporal.Instant.fromEpochNanoseconds(10n).toZonedDateTimeISO(`UTC`), + }, +] +const negativeWidthBoundary: ReadonlyArray = [ + { + id: `early`, + value: + Temporal.Instant.fromEpochNanoseconds(-10n).toZonedDateTimeISO(`UTC`), + }, + { + id: `late`, + value: Temporal.Instant.fromEpochNanoseconds(-1n).toZonedDateTimeISO(`UTC`), + }, +] +const highMagnitudeNeighbors: ReadonlyArray = [ + { + id: `z-earlier`, + value: + Temporal.Instant.fromEpochNanoseconds( + 1_704_067_200_000_000_000n, + ).toZonedDateTimeISO(`+14:00`), + }, + { + id: `a-later`, + value: + Temporal.Instant.fromEpochNanoseconds( + 1_704_067_200_000_000_001n, + ).toZonedDateTimeISO(`UTC`), + }, +] +const tiedInstants: ReadonlyArray = [ + { + id: `late`, + value: Temporal.Instant.fromEpochNanoseconds(0n).toZonedDateTimeISO(`UTC`), + }, + { + id: `early`, + value: + Temporal.Instant.fromEpochNanoseconds(0n).toZonedDateTimeISO(`+14:00`), + }, +] +const comparisons: ReadonlyArray<{ + name: string + compareOptions?: CompareOptions + expected: ReadonlyArray +}> = [ + { name: `default`, expected: expectedOrder }, + { + name: `lexical strings`, + compareOptions: { direction: `asc`, nulls: `first`, stringSort: `lexical` }, + expected: expectedOrder, + }, + { + name: `locale strings`, + compareOptions: { + direction: `asc`, + nulls: `first`, + stringSort: `locale`, + locale: `en`, + }, + expected: expectedOrder, + }, + { + name: `custom string collation`, + compareOptions: { + direction: `asc`, + nulls: `first`, + stringSort: `custom`, + compare: (a, b) => b.localeCompare(a), + }, + expected: expectedOrder, + }, + { + name: `descending`, + compareOptions: { direction: `desc`, nulls: `first` }, + expected: [`late`, `early`], + }, +] + +describe(`ZonedDateTime indexed instant order`, () => { + it(`calibrates the opposing instant and display orders`, () => { + expect(early!.value.epochNanoseconds < late!.value.epochNanoseconds).toBe( + true, + ) + expect(early!.value.toString() > late!.value.toString()).toBe(true) + expect( + widthBoundary[0]!.value.toString() > widthBoundary[1]!.value.toString(), + ).toBe(true) + expect(highMagnitudeNeighbors[0]!.value.epochNanoseconds + 1n).toBe( + highMagnitudeNeighbors[1]!.value.epochNanoseconds, + ) + expect(Number(highMagnitudeNeighbors[0]!.value.epochNanoseconds)).toBe( + Number(highMagnitudeNeighbors[1]!.value.epochNanoseconds), + ) + expect( + highMagnitudeNeighbors[0]!.value.toString() > + highMagnitudeNeighbors[1]!.value.toString(), + ).toBe(true) + expect( + [...rows] + .sort((a, b) => + a.value.epochNanoseconds < b.value.epochNanoseconds ? -1 : 1, + ) + .map((row) => row.id), + ).toEqual(expectedOrder) + expect([`late`, `early`]).not.toEqual(expectedOrder) + }) + + for (const IndexType of [BasicIndex, BTreeIndex]) { + it.each([`add`, `build`] as const)( + `${IndexType.name} %s orders the supplied instants`, + (mode) => { + const index = new IndexType(1, new PropRef([`value`])) + if (mode === `build`) index.build(rows.map((row) => [row.id, row])) + else for (const row of rows) index.add(row.id, row) + expect(index.takeFromStart(2)).toEqual(expectedOrder) + expect(index.take(1, early!.value)).toEqual([`late`]) + }, + ) + + // Equal-width 2024 epochs do not expose a key that sorts decimal text. + // The adjacent 9ns/10ns pair does, with opposing display order too. + it.each(comparisons)( + `${IndexType.name} orders instants across an epoch digit-width boundary with $name`, + ({ compareOptions, expected }) => { + const index = new IndexType( + 1, + new PropRef([`value`]), + undefined, + { compareOptions }, + ) + for (const row of widthBoundary) index.add(row.id, row) + expect(index.takeFromStart(2)).toEqual(expected) + }, + ) + + it(`${IndexType.name} orders negative instants across an epoch digit-width boundary`, () => { + const index = new IndexType(1, new PropRef([`value`])) + for (const row of negativeWidthBoundary) index.add(row.id, row) + expect(index.takeFromStart(2)).toEqual(expectedOrder) + }) + + it(`${IndexType.name} preserves adjacent nanoseconds at a large epoch`, () => { + const index = new IndexType(1, new PropRef([`value`])) + for (const row of highMagnitudeNeighbors) index.add(row.id, row) + expect(index.takeFromStart(2)).toEqual([`z-earlier`, `a-later`]) + }) + + it(`${IndexType.name} orders equal instants by key while retaining exact membership`, () => { + const index = new IndexType(1, new PropRef([`value`])) + for (const row of tiedInstants) index.add(row.id, row) + expect(index.takeFromStart(2)).toEqual(expectedOrder) + expect(index.equalityLookup(tiedInstants[0]!.value)).toEqual( + new Set([`late`]), + ) + expect(index.equalityLookup(tiedInstants[1]!.value)).toEqual( + new Set([`early`]), + ) + }) + } + + for (const [scenario, sourceRows] of [ + [`opposing display text`, rows], + [`epoch digit width`, widthBoundary], + ] as const) { + it.each([`scan`, `BasicIndex`, `BTreeIndex`] as const)( + `keeps the first public row for ${scenario} with %s`, + async (path) => { + const source = createCollection( + mockSyncCollectionOptions({ + id: `temporal-index-order-${scenario}-${path}`, + getKey: (row) => row.id, + initialData: [...sourceRows], + autoIndex: `off`, + }), + ) + let live: ReturnType | undefined + try { + await source.stateWhenReady() + const index = + path === `scan` + ? undefined + : source.createIndex((row) => row.value, { + indexType: path === `BasicIndex` ? BasicIndex : BTreeIndex, + }) + const take = index && vi.spyOn(index, `takeFromStart`) + live = createLiveQueryCollection((q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.value) + .limit(1), + ) + await live.preload() + expect([...live.values()].map((row) => row.id)).toEqual([`early`]) + if (take) expect(take).toHaveBeenCalled() + } finally { + await live?.cleanup() + await source.cleanup() + } + }, + ) + } + + for (const IndexType of [BasicIndex, BTreeIndex]) { + for (const [scenario, sourceRows, expectedFirst] of [ + [`epoch digit width`, widthBoundary, `early`], + [ + `adjacent high-magnitude nanoseconds`, + highMagnitudeNeighbors, + `z-earlier`, + ], + ] as const) { + it(`${IndexType.name} selects the earliest on-demand Collection row for ${scenario}`, async () => { + // A local ordered snapshot reads the index before asking its provider for + // more rows. The 9ns/10ns pair makes equality-key text order disagree + // with the public instant order at that synchronous publication cut. + const source = createCollection({ + id: `temporal-on-demand-${IndexType.name}-${scenario}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + autoIndex: `off`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const row of sourceRows) { + write({ type: `insert`, value: row }) + } + commit() + markReady() + return { loadSubset: () => true as const, unloadSubset: () => {} } + }, + }, + }) + let subscription: ReturnType | undefined + try { + const index = source.createIndex((row) => row.value, { + indexType: IndexType, + }) + const take = vi.spyOn(index, `takeFromStart`) + const published: Array> = [] + subscription = source.subscribeChanges( + (changes) => + published.push(changes.map((change) => change.value.id)), + { includeInitialState: false }, + ) + subscription.setOrderByIndex(index) + subscription.requestLimitedSnapshot({ + orderBy: [ + { + expression: new PropRef([`value`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + limit: 1, + }) + expect(take).toHaveBeenCalled() + expect(published).toEqual([[expectedFirst]]) + } finally { + subscription?.unsubscribe() + await source.cleanup() + } + }) + } + } +}) + +/** + * # Does an index preserve the public order of day/time Durations? + * + * Changelog #1519 makes `orderBy` use Temporal's static compare. The external + * Temporal implementation supplies this oracle's expected sign: an hour is + * shorter than 120 minutes, while an hour and 60 minutes tie in order but + * remain distinct under #1519 query equality. The fixed legal domain below + * uses day/time fields whose comparison needs no `relativeTo`. Calendar-unit + * Durations, custom index callbacks, and a public order among tied rows are + * outside the claim. + * + * Direct adds check both index engines after each supplied pair. One on-demand + * Collection then checks the first published row at requestLimitedSnapshot's + * synchronous cut. Distinct IDs deliberately reverse key order, so treating + * unequal Durations as an index tie fails the ordered observation. Exclusive + * cursors on the equal-order pair must pass neither row, even though exact + * equality lookups keep their distinct representations separate. + */ +type DurationRow = { id: string; value: Temporal.Duration } +const durationOrderCases = [ + { + name: `hours and minutes`, + earlier: Temporal.Duration.from({ hours: 1 }), + later: Temporal.Duration.from({ minutes: 120 }), + }, + { + name: `negative hours and minutes`, + earlier: Temporal.Duration.from({ hours: -2 }), + later: Temporal.Duration.from({ minutes: -30 }), + }, + { + name: `days and hours`, + earlier: Temporal.Duration.from({ hours: 23 }), + later: Temporal.Duration.from({ days: 1 }), + }, + { + name: `minutes and seconds`, + earlier: Temporal.Duration.from({ seconds: 59 }), + later: Temporal.Duration.from({ minutes: 1 }), + }, + { + name: `seconds and milliseconds`, + earlier: Temporal.Duration.from({ milliseconds: 999 }), + later: Temporal.Duration.from({ seconds: 1 }), + }, + { + name: `milliseconds and microseconds`, + earlier: Temporal.Duration.from({ microseconds: 999 }), + later: Temporal.Duration.from({ milliseconds: 1 }), + }, + { + name: `microseconds and nanoseconds`, + earlier: Temporal.Duration.from({ nanoseconds: 999 }), + later: Temporal.Duration.from({ microseconds: 1 }), + }, +] +const publishedDurationRows: ReadonlyArray = [ + { id: `z-shorter`, value: Temporal.Duration.from({ hours: 1 }) }, + { id: `a-longer`, value: Temporal.Duration.from({ minutes: 120 }) }, +] + +describe(`Duration indexed time order`, () => { + it.each(durationOrderCases)( + `calibrates the $name order against Temporal`, + ({ earlier, later }) => { + expect(Temporal.Duration.compare(earlier, later)).toBeLessThan(0) + }, + ) + + for (const IndexType of [BasicIndex, BTreeIndex]) { + it.each(durationOrderCases)( + `${IndexType.name} preserves $name order`, + ({ earlier, later }) => { + const index = new IndexType(1, new PropRef([`value`])) + index.add(`z-earlier`, { value: earlier }) + index.add(`a-later`, { value: later }) + expect(index.takeFromStart(2)).toEqual([`z-earlier`, `a-later`]) + }, + ) + + it(`${IndexType.name} retains distinct exact Duration values at an order tie`, () => { + const hour = Temporal.Duration.from({ hours: 1 }) + const minutes = Temporal.Duration.from({ minutes: 60 }) + expect(Temporal.Duration.compare(hour, minutes)).toBe(0) + const index = new IndexType(1, new PropRef([`value`])) + index.add(`z-hour`, { value: hour }) + index.add(`a-minutes`, { value: minutes }) + expect(index.equalityLookup(hour)).toEqual(new Set([`z-hour`])) + expect(index.equalityLookup(minutes)).toEqual(new Set([`a-minutes`])) + expect(new Set(index.takeFromStart(2))).toEqual( + new Set([`z-hour`, `a-minutes`]), + ) + expect(index.take(2, minutes)).toEqual([]) + expect(index.takeReversed(2, hour)).toEqual([]) + }) + + it(`${IndexType.name} publishes the shorter Duration in an on-demand ordered snapshot`, async () => { + const source = createCollection({ + id: `duration-on-demand-${IndexType.name}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + autoIndex: `off`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const row of publishedDurationRows) { + write({ type: `insert`, value: row }) + } + commit() + markReady() + return { loadSubset: () => true as const, unloadSubset: () => {} } + }, + }, + }) + let subscription: ReturnType | undefined + try { + const index = source.createIndex((row) => row.value, { + indexType: IndexType, + }) + const take = vi.spyOn(index, `takeFromStart`) + const published: Array> = [] + subscription = source.subscribeChanges( + (changes) => published.push(changes.map((change) => change.value.id)), + { includeInitialState: false }, + ) + subscription.setOrderByIndex(index) + subscription.requestLimitedSnapshot({ + orderBy: [ + { + expression: new PropRef([`value`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + limit: 1, + }) + expect(take).toHaveBeenCalled() + expect(published).toEqual([[`z-shorter`]]) + } finally { + subscription?.unsubscribe() + await source.cleanup() + } + }) + } +}) + +/** + * # Does string collation change PlainDate calendar order? + * + * Changelog #1519 promises calendar order for PlainDate. With equal month and + * day, the year decides: -100000 precedes +10000. Their ISO display strings + * have the opposite lexical order, so sorting normalized text would choose the + * wrong first row. String collation applies to strings, not Temporal values. + * + * A ready two-row source is read through a scan, BasicIndex, or BTreeIndex. + * The model reads the supplied years, while direct ordered snapshots and + * live-query preload expose the public first row. Index spies establish the + * direct indexed reads; preload is a parity check. This fixed history does + * not claim all extended-year pairs or on-demand publication. + */ +describe(`PlainDate extended-year calendar order`, () => { + const calendarRows = [ + { id: `negative`, value: Temporal.PlainDate.from(`-100000-01-01`) }, + { id: `positive`, value: Temporal.PlainDate.from(`+010000-01-01`) }, + ] + const lexicalOrder = { + direction: `asc` as const, + nulls: `first` as const, + stringSort: `lexical` as const, + } + + it(`calibrates opposing year and display orders`, () => { + expect(calendarRows[0]!.value.year).toBeLessThan( + calendarRows[1]!.value.year, + ) + expect( + calendarRows[0]!.value.toString() > calendarRows[1]!.value.toString(), + ).toBe(true) + }) + + it.each([`scan`, `BasicIndex`, `BTreeIndex`] as const)( + `keeps calendar order with %s`, + async (path) => { + const source = createCollection( + mockSyncCollectionOptions({ + id: `plain-date-extended-year-${path}`, + getKey: (row: (typeof calendarRows)[number]) => row.id, + initialData: calendarRows, + autoIndex: `off`, + }), + ) + let live: ReturnType | undefined + await withOracleCleanup(async () => { + await source.stateWhenReady() + const index = + path === `scan` + ? undefined + : source.createIndex((row) => row.value, { + indexType: path === `BasicIndex` ? BasicIndex : BTreeIndex, + options: { compareOptions: lexicalOrder }, + }) + const take = index && vi.spyOn(index, `takeFromStart`) + try { + const direct = source.currentStateAsChanges({ + orderBy: [ + { + expression: new PropRef([`value`]), + compareOptions: lexicalOrder, + }, + ], + limit: 1, + })! + expect(direct.map((change) => change.value.id)).toEqual([`negative`]) + if (take) expect(take).toHaveBeenCalled() + + live = createLiveQueryCollection((q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.value, lexicalOrder) + .limit(1), + ) + await live.preload() + expect([...live.values()].map((row) => row.id)).toEqual([`negative`]) + } finally { + take?.mockRestore() + } + }, [() => live?.cleanup(), () => source.cleanup()]) + }, + ) +}) + +/** + * # Can an index hide an undefined public Temporal order? + * + * Changelog #1519 requires TanStack DB to reject ordering two different + * Temporal kinds; PlainMonthDay has no static comparison. The model therefore + * predicts a TypeError for either two-row ordering, regardless of which row + * would be first. Index storage may assign a physical position to both values, + * but that position cannot turn this query into a successful public read. + * + * Each fixed history starts a ready source with both rows, optionally creates + * a BasicIndex or BTreeIndex, then checks the direct ordered snapshot and a + * live-query preload. These are public observation cuts; the oracle does not + * claim that a later invalid update leaves a usable Collection or that an + * on-demand provider has a matching error boundary. + */ +type UnorderedTemporalRow = { + id: string + value: Temporal.PlainDate | Temporal.PlainDateTime | Temporal.PlainMonthDay +} +const unorderedTemporalCases: ReadonlyArray<{ + name: string + rows: Array +}> = [ + { + name: `different Temporal kinds`, + rows: [ + { id: `date`, value: Temporal.PlainDate.from(`2024-01-01`) }, + { + id: `date-time`, + value: Temporal.PlainDateTime.from(`2024-01-02T00:00`), + }, + ], + }, + { + name: `PlainMonthDay without an order`, + rows: [ + { id: `january`, value: Temporal.PlainMonthDay.from(`01-01`) }, + { id: `february`, value: Temporal.PlainMonthDay.from(`02-01`) }, + ], + }, +] + +describe(`undefined Temporal public order`, () => { + it.each(unorderedTemporalCases)( + `calibrates the $name premise`, + ({ name, rows: sampleRows }) => { + if (name === `different Temporal kinds`) { + expect(Object.prototype.toString.call(sampleRows[0]!.value)).not.toBe( + Object.prototype.toString.call(sampleRows[1]!.value), + ) + } else { + expect(Reflect.get(Temporal.PlainMonthDay, `compare`)).toBeUndefined() + } + }, + ) + + for (const scenario of unorderedTemporalCases) { + for (const path of [`scan`, `BasicIndex`, `BTreeIndex`] as const) { + it(`${path} keeps the ${scenario.name} public error`, async () => { + const source = createCollection( + mockSyncCollectionOptions({ + id: `undefined-temporal-order-${scenario.name}-${path}`, + getKey: (row) => row.id, + initialData: scenario.rows, + autoIndex: `off`, + }), + ) + let live: ReturnType | undefined + await withOracleCleanup(async () => { + await source.stateWhenReady() + if (path !== `scan`) { + source.createIndex((row) => row.value, { + indexType: path === `BasicIndex` ? BasicIndex : BTreeIndex, + }) + } + expect(() => + source.currentStateAsChanges({ + orderBy: [ + { + expression: new PropRef([`value`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + limit: 1, + }), + ).toThrow(TypeError) + live = createLiveQueryCollection((q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.value) + .limit(1), + ) + await expect(live.preload()).rejects.toThrow(TypeError) + }, [() => live?.cleanup(), () => source.cleanup()]) + }) + } + } +}) diff --git a/packages/db/tests/utils-oracle.property.test.ts b/packages/db/tests/utils-oracle.property.test.ts index c4cda036c1..2ad77a4632 100644 --- a/packages/db/tests/utils-oracle.property.test.ts +++ b/packages/db/tests/utils-oracle.property.test.ts @@ -8,12 +8,16 @@ import { deepEquals, equalPersistedSnapshotValues } from '../src/utils' * established examples in `utils.test.ts`, promises change-event equality for * its supported value classes. This oracle checks bounded equality and * inequality laws for those classes, not general graph isomorphism. - * * Independent value constructors exercise reflexivity, symmetry, copied * structure, and changed-leaf inequality for primitives, arrays, records, - * dates, regexes, typed bytes, and selected Temporal types. Cross-type values - * remain unequal. Separate bounded graph laws construct corresponding rings - * and acyclic copies without reusing the production walk. + * dates, regexes, typed bytes, and selected Temporal types. Named cross-type + * pairs remain unequal, even when a primitive displays or encodes a Date or + * RegExp value. Published #1519 keeps equivalent Temporal.Duration forms + * distinct for query `eq`. The `deepEquals` API and #1901's change-message + * reconstruction law carry that field distinction to Collection events: a + * changed public Duration row cannot be suppressed because its display text + * matches. Separate bounded graph laws construct corresponding rings and + * acyclic copies without reusing the production walk. * * Arbitrary different cycle topologies are outside this model. Naming that * exclusion matters: more random examples cannot establish semantics the @@ -88,6 +92,9 @@ const arbitraryPrimitive = fc.oneof( ) const arbitraryDate = fc.date().map((d) => new Date(d.getTime())) +const arbitraryFiniteDate = fc + .integer({ min: -1_000_000_000_000, max: 2_000_000_000_000 }) + .map((time) => new Date(time)) const arbitraryRegExp = fc .tuple(fc.string(), fc.constantFrom(``, `g`, `i`, `gi`, `m`, `gim`)) @@ -120,6 +127,30 @@ const arbitraryTemporalDuration = fc Temporal.Duration.from({ hours, minutes, seconds }), ) +// One nonzero intrinsic field at a time is a legal Duration. Every campaign +// enumerates all ten fields at generated magnitudes 1–9 and both signs. Each +// field has an equal fresh copy, a changed-field neighbor, and an opposite-sign +// value, so its exact value affects change-event equality independently of +// production comparison. +const durationFields = [ + `years`, + `months`, + `weeks`, + `days`, + `hours`, + `minutes`, + `seconds`, + `milliseconds`, + `microseconds`, + `nanoseconds`, +] as const + +const adjacentDurationUnits = [ + [`seconds`, `milliseconds`], + [`milliseconds`, `microseconds`], + [`microseconds`, `nanoseconds`], +] as const + // Same-type value arbitraries for testing equivalence properties // Mixed Date/Temporal inequality has its own generated and fixed witnesses below. const arbitrarySameTypePrimitive = fc.oneof( @@ -190,8 +221,9 @@ const arbitraryDefinedValue = arbitrarySingleValue.filter( // independently generated values, so both equal and unequal pairs are legal. // The fixed witnesses exercise those two outcomes; deleting either outcome // would leave an always-true or always-false comparator undetected. The -// distinct-type grammar is deliberately restricted to Date/Temporal pairs; -// it does not claim every cross-class combination. String lengths, integer +// generated distinct-type grammar names Date/Temporal pairs, Date versus its +// ISO text and epoch number, and RegExp versus its source text; it does not +// claim every cross-class combination. String lengths, integer // magnitude, and object keys are fast-check defaults; explicit array/record // bounds are 0–5 for equivalence and 0–10 for structural copies, bytes 0–50, // Float32Array 0–10, Temporal years 1–9999/months 1–12/days 1–28, and @@ -691,6 +723,53 @@ describe(`deepEquals property-based tests`, () => { }) describe(`cross-type comparisons`, () => { + // A Date's display and numeric timestamp encode the same instant, but + // change-event equality keeps the Date value class distinct. The finite + // generator makes both encodings legal; the 2023 case preserves the old + // unit test's exact operands. + it(`keeps a 2023 Date distinct from its old primitive probes`, () => { + const date = new Date(`2023-01-01T00:00:00Z`) + expectCopyAndChanged( + date, + new Date(`2023-01-01T00:00:00Z`), + new Date(`2023-01-02T00:00:00Z`), + ) + assertDistinctTypes(date, `2023-01-01T00:00:00Z`, deepEquals) + assertDistinctTypes(date, date.getTime(), deepEquals) + }) + + // RegExp source text does not acquire the regex's value class, even when + // it spells the same pattern. The source and flags still govern equality + // between two genuine RegExps in the laws below. + it(`keeps /abc/g distinct from its old source-text probe`, () => { + expectCopyAndChanged(/abc/g, /abc/g, /abc/i) + expectCopyAndChanged(/abc/g, /abc/g, /def/g) + expectEqualityPair(/test/i, /test/i, true) + assertDistinctTypes(/abc/g, `abc`, deepEquals) + }) + + it(`rejects primitive-coercing equality for Date and RegExp`, () => { + const date = new Date(`2023-01-01T00:00:00Z`) + const dateAsNumber = (left: unknown, right: unknown) => + (left instanceof Date ? left.getTime() : left) === + (right instanceof Date ? right.getTime() : right) + const dateAsString = (left: unknown, right: unknown) => + (left instanceof Date ? left.toISOString() : left) === + (right instanceof Date ? right.toISOString() : right) + const regexpAsSource = (left: unknown, right: unknown) => + (left instanceof RegExp ? left.source : left) === + (right instanceof RegExp ? right.source : right) + expect(() => + assertDistinctTypes(date, date.getTime(), dateAsNumber), + ).toThrow(`distinct types must be unequal (forward)`) + expect(() => + assertDistinctTypes(date, date.toISOString(), dateAsString), + ).toThrow(`distinct types must be unequal (forward)`) + expect(() => assertDistinctTypes(/abc/g, `abc`, regexpAsSource)).toThrow( + `distinct types must be unequal (forward)`, + ) + }) + utilsProperty([arbitraryDate, arbitraryTemporalDuration])( `generated Date and Temporal.Duration are unequal in both directions`, (date, duration) => assertDistinctTypes(date, duration, deepEquals), @@ -1129,11 +1208,15 @@ describe(`deepEquals property-based tests`, () => { const changedFlags = regex.flags.includes(`i`) ? regex.flags.replace(`i`, ``) : `${regex.flags}i` + // The source neighbor keeps flags fixed; a source-blind comparator + // must fail even when generated flags happen to match. + const changedSource = new RegExp(`${regex.source}(?:x)`, regex.flags) expectCopyAndChanged( regex, copy, new RegExp(regex.source, changedFlags), ) + expectEqualityPair(regex, changedSource, false) }, ) @@ -1164,6 +1247,69 @@ describe(`deepEquals property-based tests`, () => { expectCopyAndChanged(duration, copy, changed) }, ) + + utilsProperty([arbitraryFiniteDate])( + `Date differs from its ISO text and epoch number in both directions`, + (date) => { + assertDistinctTypes(date, date.toISOString(), deepEquals) + assertDistinctTypes(date, date.getTime(), deepEquals) + }, + ) + + utilsProperty([arbitraryRegExp])( + `RegExp differs from its source text in both directions`, + (regex) => assertDistinctTypes(regex, regex.source, deepEquals), + ) + + // The source fields, rather than Temporal's display or ordered duration, + // form the reference identity. The production path is deepEquals in both + // directions; its direct result is compared after each constructed pair. + // Every field is exercised at a positive magnitude, its neighbor, and its + // negative counterpart. No one field may be ignored or compared by magnitude. + utilsProperty([fc.integer({ min: 1, max: 9 })])( + `each Temporal.Duration field determines change-event equality`, + (value) => { + for (const field of durationFields) { + const original = Temporal.Duration.from({ [field]: value }) + const copy = Temporal.Duration.from({ [field]: value }) + const changed = Temporal.Duration.from({ [field]: value + 1 }) + const opposite = Temporal.Duration.from({ [field]: -value }) + expectCopyAndChanged(original, copy, changed) + expectEqualityPair(original, opposite, false) + } + }, + ) + + it(`keeps one second distinct from 1000 milliseconds with the same display`, () => { + const seconds = Temporal.Duration.from({ seconds: 1 }) + const milliseconds = Temporal.Duration.from({ milliseconds: 1000 }) + expect(seconds.toString()).toBe(milliseconds.toString()) + expectEqualityPair(seconds, milliseconds, false) + }) + + // This generated grammar varies magnitude and sign while enumerating every + // adjacent subsecond unit boundary. Both constructors encode the same + // displayed elapsed time, but the single nonzero source field differs. + // #1519's query equality distinction and #1901's event reconstruction + // law together predict inequality in both directions here. This rejects + // a comparator that checks seconds alone before using display text. + utilsProperty([fc.integer({ min: 1, max: 9 })])( + `adjacent Duration units remain unequal despite matching display`, + (magnitude) => { + for (const sign of [-1, 1] as const) { + for (const [coarser, finer] of adjacentDurationUnits) { + const left = Temporal.Duration.from({ + [coarser]: sign * magnitude, + }) + const right = Temporal.Duration.from({ + [finer]: sign * magnitude * 1000, + }) + expect(left.toString()).toBe(right.toString()) + expectEqualityPair(left, right, false) + } + } + }, + ) }) }) diff --git a/packages/db/tests/utils.test.ts b/packages/db/tests/utils.test.ts index acffea3b69..1bbf4e4730 100644 --- a/packages/db/tests/utils.test.ts +++ b/packages/db/tests/utils.test.ts @@ -288,42 +288,6 @@ describe(`deepEquals`, () => { }) }) - describe(`Date objects`, () => { - it(`should handle identical dates`, () => { - const date1 = new Date(`2023-01-01T00:00:00Z`) - const date2 = new Date(`2023-01-01T00:00:00Z`) - expect(deepEquals(date1, date2)).toBe(true) - }) - - it(`should handle different dates`, () => { - const date1 = new Date(`2023-01-01T00:00:00Z`) - const date2 = new Date(`2023-01-02T00:00:00Z`) - expect(deepEquals(date1, date2)).toBe(false) - }) - - it(`should handle date vs non-date`, () => { - const date = new Date(`2023-01-01T00:00:00Z`) - expect(deepEquals(date, `2023-01-01T00:00:00Z`)).toBe(false) - expect(deepEquals(date, date.getTime())).toBe(false) - }) - }) - - describe(`RegExp objects`, () => { - it(`should handle identical regexes`, () => { - expect(deepEquals(/abc/g, /abc/g)).toBe(true) - expect(deepEquals(/test/i, /test/i)).toBe(true) - }) - - it(`should handle different regexes`, () => { - expect(deepEquals(/abc/g, /abc/i)).toBe(false) - expect(deepEquals(/abc/g, /def/g)).toBe(false) - }) - - it(`should handle regex vs non-regex`, () => { - expect(deepEquals(/abc/g, `abc`)).toBe(false) - }) - }) - describe(`Map objects`, () => { it(`should handle identical maps`, () => { const map1 = new Map([ diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index c952b7674e..54267feecc 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -2,7 +2,7 @@ import { expect } from 'vitest' import { createCollection } from '../src/collection/index.js' import { BTreeIndex } from '../src/indexes/btree-index' import { withCollectionConfigFactory } from '../src/client' -import { denormalizeUndefined } from '../src/utils/comparison.js' +import { normalizeValue } from '../src/utils/comparison.js' import { CleanupQueue } from '../src/collection/cleanup-queue.js' import type { CollectionConfig, @@ -601,16 +601,25 @@ export function withExpectedRejection( type IndexInternals = { indexedKeys: Set } & ( | { sortedValues: Array; valueMap: Map> } | { - valueMap: Map }> + valueMap: Map< + unknown, + { + keys: Map + buckets?: Map> + } + > orderedEntries: { size: number - minKey: () => unknown - maxKey: () => unknown + minKey: () => { value: unknown } + maxKey: () => { value: unknown } forRange: ( low: unknown, high: unknown, includeHigh: boolean, - onFound: (key: unknown, bucket: { keys: Set }) => void, + onFound: ( + key: { value: unknown }, + group: { keys: Map }, + ) => void, ) => void } } @@ -632,12 +641,14 @@ export function indexedKeysSet(index: object): Set { } /** Test inspection of an index's value buckets keyed by indexed value. */ -export function valueMapData(index: object): Map> { +export function valueMapData( + index: object, +): Map, 'size' | 'has'>> { const [internals] = indexInternals(index) if (`sortedValues` in internals) return internals.valueMap - const result = new Map>() - for (const [key, bucket] of internals.valueMap) { - result.set(denormalizeUndefined(key), bucket.keys) + const result = new Map, 'size' | 'has'>>() + for (const [key, group] of internals.valueMap) { + result.set(key, group.buckets?.get(key) ?? group.keys) } return result } @@ -651,14 +662,14 @@ export function orderedEntriesArray( if (`sortedValues` in internals) { entries = internals.sortedValues.map((value) => [ value, - internals.valueMap.get(value) ?? new Set(), + internals.valueMap.get(normalizeValue(value)) ?? new Set(), ]) } else { const tree = internals.orderedEntries entries = [] if (tree.size > 0) { - tree.forRange(tree.minKey(), tree.maxKey(), true, (key, bucket) => { - entries.push([denormalizeUndefined(key), bucket.keys]) + tree.forRange(tree.minKey(), tree.maxKey(), true, (key, group) => { + entries.push([key.value, new Set(group.keys.keys())]) }) } }