From 58befb7dc6c3690707047b45af09e5760ead2325 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:32:20 +0530 Subject: [PATCH 01/23] storage: key the decoded tables by device, not just time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decoded_onehz was rec_ts PRIMARY KEY with REPLACE, and _queueRrBeats deleted the whole second before inserting. a second band at the same second didn't merge, it deleted the first one's row and beats, and raw prunes at 3 days so it's gone. same shape in samples, keyed on a whoop flash counter. now (device_id, ts_ms). ts_ms is rec_ts*1000 exactly — never plus the subsecond — so the key stays as unique as rec_ts was and the newest-wins dedupe is unchanged. device_id = '' is reserved permanently for the primary band; only secondaries get a real id, because an ios peripheral uuid or an android rpa would otherwise split one band into many. rec_ts stays as an indexed column so no read changed. two things bricked the ladder before this worked: two mid-ladder replays run before the v47 rung and named a column that doesn't exist yet, which throws inside onUpgrade and quarantines the db. and the importer would have taken the defaults and REPLACE'd a whole export down to one row. also here: beat_ts_ms was written and never read, so every dropout under a second was invisible and got spliced out of the time axis. rr beats with no matching frame row were silently dropped. hrSamplesInRange was the one decoded reader missing the source filter. two scanners each stopped the other's scan and then awaited it, so a scan could report "found nothing" with no error. observation table for what a band computes itself, isolated from every baseline. algo 77. --- lib/ble/ble_engine.dart | 10 +- lib/ble/ble_state.dart | 34 + lib/ble/hr_sensor.dart | 21 +- lib/compute/derivation_engine.dart | 155 +++- lib/compute/derive_prepare.dart | 118 ++- lib/compute/onehz_pipeline.dart | 31 +- lib/compute/substrate.dart | 245 +++++- lib/data/db.dart | 699 +++++++++++++++--- lib/data/live_coverage_policy.dart | 43 +- lib/data/local_repository_impl.dart | 3 +- lib/data/models.dart | 1 - lib/data/observation.dart | 95 +++ test/absence_and_offload_guards_test.dart | 20 +- test/band_step_counter_test.dart | 29 +- test/bandagnostic_c10_c15_test.dart | 206 ++++++ test/beat_clock_read_path_test.dart | 150 ++++ test/beat_timestamps_test.dart | 21 +- test/ble_state_test.dart | 37 + test/cadence_decimation_rig_test.dart | 504 +++++++++++++ test/cadence_group_c_nocturnal_rig_test.dart | 168 +++++ test/db_integrity_test.dart | 4 + test/db_migration_ladder_test.dart | 299 +++++++- test/db_serve_version_and_reads_test.dart | 2 + test/db_storage_hygiene_test.dart | 25 +- .../db_v42_retention_and_provenance_test.dart | 148 ++++ test/db_v43_nullable_hr_test.dart | 5 +- test/night_beats_repo_test.dart | 3 + test/observation_isolation_test.dart | 492 ++++++++++++ test/step_source_ladder_test.dart | 9 +- test/substrate_hr_valid_test.dart | 15 +- 30 files changed, 3344 insertions(+), 248 deletions(-) create mode 100644 lib/data/observation.dart create mode 100644 test/bandagnostic_c10_c15_test.dart create mode 100644 test/beat_clock_read_path_test.dart create mode 100644 test/cadence_decimation_rig_test.dart create mode 100644 test/cadence_group_c_nocturnal_rig_test.dart create mode 100644 test/observation_isolation_test.dart diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index afa7dc1f..9d56505f 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -1719,9 +1719,17 @@ class BleEngine { /// Service-filtered scan (mandatory on iOS/macOS — passive scans hide the UUID). /// Start ONE scan, stop early on a match, otherwise let the timeout stop it. /// NEVER rapid start/stop (Android throttles → SCANNING_TOO_FREQUENTLY). + /// + /// Serialised process-wide through [withScanLock]: the HR-sensor scan shares + /// this one radio scanner, and the `isScanning == false` await below is + /// satisfied by ITS `stopScan` too — an unserialised scan silently ends + /// having seen nothing and reports "No WHOOP found". Future scan({ Duration timeout = const Duration(seconds: 12), - }) async { + }) => + withScanLock(() => _scanLocked(timeout)); + + Future _scanLocked(Duration timeout) async { // A phone-level blocker is NOT "nothing answered". Returning null for a // revoked Bluetooth permission classified it as `notFound` upstream, which // told the user to walk closer to a band that was never the problem — the diff --git a/lib/ble/ble_state.dart b/lib/ble/ble_state.dart index a9149fd5..e9566311 100644 --- a/lib/ble/ble_state.dart +++ b/lib/ble/ble_state.dart @@ -1688,3 +1688,37 @@ class BatteryPackInfoGate { } } +/// Process-wide BLE scan mutex. +/// +/// The radio has ONE scanner and this app has two callers of it: the band scan +/// (`BleEngine.scan`) and the heart-rate sensor scan (`HrSensorLink.scan`). +/// Both stop whatever is already scanning and then wait for +/// `FlutterBluePlus.isScanning` to go false — an await that any scan stopping +/// satisfies, including the OTHER caller's. The loser's scan therefore +/// "completed" the instant the winner called `stopScan`, having seen nothing, +/// and the caller reported "no device found" with no error anywhere to say +/// why. Serialising the two bodies is the whole fix. +/// +/// Same idiom as `BleEngine._locked` — chain onto the previous op, hand the +/// caller a completer, swallow nothing — hoisted out of the engine because the +/// contention is BETWEEN objects, not between two ops on one engine. It has to +/// stay static once a primary band and a secondary device can both scan. +/// +/// ponytail: one global lock, so a queued scan waits out the running one's +/// whole timeout (~12 s) rather than joining it. Give waiters the running +/// scan's results only if a screen ever needs both scans at once. +Future withScanLock(Future Function() body) { + final completer = Completer(); + // A body that throws must not break the chain — the error goes to its own + // caller and the next scan still runs. + _scanLock = _scanLock.then((_) async { + try { + completer.complete(await body()); + } catch (e, st) { + completer.completeError(e, st); + } + }); + return completer.future; +} + +Future _scanLock = Future.value(); diff --git a/lib/ble/hr_sensor.dart b/lib/ble/hr_sensor.dart index 20de8059..1abac51e 100644 --- a/lib/ble/hr_sensor.dart +++ b/lib/ble/hr_sensor.dart @@ -38,6 +38,7 @@ import 'package:shared_preferences/shared_preferences.dart'; import '../data/db.dart'; import '../sync/paired_device.dart' show cleanDeviceLabel; +import 'ble_state.dart' show withScanLock; /// GATT Heart Rate Service and its Heart Rate Measurement characteristic. /// Written out in full 128-bit form rather than the 16-bit shorthand: the @@ -183,9 +184,17 @@ class HrSensorLink { /// on the workout path — it shares one radio scanner with the band's own scan. /// /// Returns (remoteId, name) pairs, deduplicated, strongest first. + /// + /// Serialised process-wide through [withScanLock] against the band's scan: + /// one radio scanner, and the `isScanning == false` await below completes on + /// the OTHER scan's `stopScan` — which ends this one early with an empty + /// list that looks exactly like "no sensor is nearby". Future> scan({ Duration timeout = const Duration(seconds: 8), - }) async { + }) => + withScanLock(() => _scanLocked(timeout)); + + Future> _scanLocked(Duration timeout) async { if (FlutterBluePlus.isScanningNow) await FlutterBluePlus.stopScan(); final found = {}; final sub = FlutterBluePlus.onScanResults.listen((results) { @@ -220,7 +229,15 @@ class HrSensorLink { /// Never scans: it connects straight to the stored remote id, so arming a /// workout cannot contend with the band's scan. Future arm(String sessionId) async { - if (armed.value) return true; + if (armed.value) { + // Re-arm on a link that is already up: keep the link, REBIND the session. + // Returning early without this filed every beat of the next workout under + // the previous workout's id. Nothing needs flushing first — `_onValue` + // stamps each buffered row with the id it was recorded under, so rows + // already queued keep pointing at the session they belong to. + _sessionId = sessionId; + return true; + } final paired = await PairedHrSensor.load(); if (paired == null) return false; _sessionId = sessionId; diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index d1ccd8ea..72a55e98 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1014,11 +1014,12 @@ import 'substrate.dart'; // already gated `hr > 0`), but a new query must not assume NOT NULL. // // 6. FROM ANALYTICS (98d42b6, the device-seam commit — and only that): -// * `device.dart`: `DeviceFamily`, `deviceFamilyOf()` returning null for -// anything it does not recognise, and `calibrationFor()` returning null -// rather than handing back gen4's constants. This is what (1) dispatches -// on. No registry, no shared constants file — a metric declares its own -// map next to itself and refuses when the family is not in it. +// * `device.dart`: `calibrationFor()` returning null for an unstamped +// strap, or for one this metric has no numbers for, rather than handing +// back gen4's constants. This is what (1) dispatches on. No registry, no +// shared constants file and no closed list of families — a metric +// declares its own String-keyed map next to itself and refuses when the +// stamp is not a key in it. // * journal correlations ran 9 numeric fields against 4 outcomes behind a // per-test gate. That is 36 simultaneous tests, so ~2 spurious // "meaningful" findings per user were guaranteed by construction. @@ -1287,7 +1288,49 @@ import 'substrate.dart'; // imported this is a strict no-op: the set is empty and every read is unchanged. // Days already finalized keep the score they were derived with — raw is pruned, // so no bump can heal them. -const int kAlgoVersion = 76; +// v77 — BAND-AGNOSTIC GROUPS A AND C. The 1 Hz assumption comes out of the +// metrics that were counting array positions and calling them seconds, and four +// gen4 defects found on the way out. Seven things move a published number: +// 1. A1 — BEAT TIMES REACH THE READ PATH. `decoded_rr.beat_ts_ms` (the +// record's own sub-second anchor) was written and never SELECTed, so every +// beat sat on a whole-second staircase and no dropout under 1 s was +// visible. RMSSD +0.031% and pNN50 +0.63% over a real 27,114-beat block; +// LF/HF and `span_sec` move with the re-anchored axis. SDNN does not — it +// never pairs successive beats. +// 2. A2 — ACCEL VALIDITY SURVIVES SEGMENTATION. `AccelSample.valid` was +// dropped converting to the segmenter's own type, so a night the strap +// never vouched for was staged as if it had. Measured: 8 h of +// `valid:false` published `TST 28619s, efficiency 100.0%`; it is absent +// now. +// 3. A3 — BEATS WITH NO FRAME ARE KEPT. RR was looked up only inside the +// frames loop, so a beat whose second has no `decoded_onehz` row was +// discarded — and gen4's hr-only historical R10-lite produces exactly +// those. The 1 Hz substrate is the UNION of frame and beat seconds now. +// 4. A4 — READINESS NEEDS 14 NIGHTS, not 3. A 3-night baseline is three +// numbers: its MAD is routinely one quantum wide or zero, which is how a +// degenerate z reached the logistic and the ring flashed a rail. Sub- +// quantum spreads are refused outright rather than z-scored. +// 5. C1 — THE 30-MINUTE TROUGH IS 30 MINUTES. `nocturnalRhr` walks a wall +// clock (`onehz_pipeline.dart`, `tsSec: sleepTs`) instead of counting 1800 +// positions. At a perfectly dense 1 Hz the two coincide; on real nights +// they do not, because real nights have holes. Measured over this owner's +// export: 2026-08-13 and 2026-08-14 are bit-identical (1 and 243 missing +// seconds), 2026-08-12 moves 62.952 → 62.857 bpm (506 missing seconds of +// 25,262 — the positional window was averaging over ~30.5 min of clock and +// missing the real trough). The wall-clock answer is the correct one, and +// the same edit is what lets the metric exist at all at 60 s and 300 s, +// where it previously abstained. It feeds readiness's 0.30-weight driver. +// 6. C9 — no metric credits a sample whose cadence cannot be measured. One +// `sampleCadenceSeconds` with an ABSTENTION replaces three median-interval +// helpers with numeric fallbacks; a band slower than 300 s used to have +// every reading credited as one second (~300x zone undercount) and now has +// no zone figure at all. +// 7. C13 — an unstamped substrate carrying a step counter reports +// `band_measured: null`. Without a device stamp there is no modulus to +// unwrap it with, and a guessed one publishes a step count at tier HIGH. +// Days already finalized keep the score they were derived with — raw prunes at +// `rawRetentionDays`, so no bump can heal them. +const int kAlgoVersion = 77; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. @@ -1334,6 +1377,19 @@ const int kAlgoVersion = 76; // COMMAND builders (alarmRev1Payload and friends) plus their exports and // test. Commands go TO the strap; no decoder line moves, so no stored number // can. +// +// v77 BUMPS WITH THE ANALYTICS PIN UNMOVED, AND THAT IS A DEBT, NOT A CLAIM. +// Four of the seven items above (A2, A4, C1's window, C9) live in analytics and +// are UNCOMMITTED — `pubspec_overrides.yaml` points the local build at +// `../analytics`, so a dev build already runs them while `d9362a6` (the pin, +// and the sibling's current HEAD) does not contain a line of them. No SHA is +// invented here: there is nothing to pin to yet. THE PIN MUST MOVE IN THE SAME +// CHANGE THAT COMMITS ANALYTICS, before anything is released at v77 — a build +// that ships this constant against the old pin serves days derived from +// different maths under one version number, which is exactly the v67/v68 +// failure this block exists to stop. The guard test cannot catch it: it +// compares this constant to pubspec.yaml, and both are consistent right now. +// ponytail: unpinned sibling, upgrade = repin at the analytics commit. const String kAnalyticsPin = 'd9362a66fbeac326d5d7d7b1fe27b28e41169a79'; const String kProtocolPin = '4ce8f021568a4cd9a1d86c91004f91c6b21980da'; @@ -1527,16 +1583,28 @@ class _BaselineHistoryCache { /// /// The mask is taken ONCE per load and applied to every key, because the /// query behind it scans day bundles (see [LocalDb.importedDates]). + /// + /// A SECOND, NARROWER MASK rides along: days measured by a different BAND + /// FAMILY, applied only to [LocalDb.familySeamKeys]. That one is not about a + /// foreign algorithm — it is this app's own maths over a sensor whose units + /// differ (`skin_temp_adc` is a gen4 ADC count on one side of a strap swap + /// and a gen5 centi-degree on the other), so it is masked PER-METRIC rather + /// than blanket: a family seam that only makes a number noisier (RHR, RMSSD) + /// stays in the window. Empty for a user who has never changed generation, + /// which is why it moves no number today. static Future<_BaselineHistoryCache> load() async { final imported = await LocalDb.importedDates(); + final foreignFamily = await LocalDb.foreignFamilyDates(); Future> hist(String key) async { final rows = await LocalDb.metricSeries(key); + final seam = + LocalDb.familySeamKeys.contains(key) ? foreignFamily : const {}; final out = <_DatedValue>[]; for (final row in rows) { final date = row['date']; final value = row['value']; if (date is! String || date.isEmpty || value is! num) continue; - if (imported.contains(date)) continue; + if (imported.contains(date) || seam.contains(date)) continue; out.add((date: date, value: value.toDouble())); } return out; @@ -2636,6 +2704,16 @@ class DerivationEngine { int? afterCursor; var rangePages = 0; var rangeRows = 0; + // Low water mark for the BEAT read, which is NOT the same window as the + // frame read. Paging is driven by `decoded_onehz`, but a beat can exist + // for a second with no 1 Hz row at all (gen4 R10-lite is hr-only and + // stays out of that table — see db.dart `_queueDecodedOneHz`). Clamping + // each page's beat read to that page's own first/last frame second + // dropped every such beat that fell before the day's first frame, after + // its last, or in the seam between two pages. Each page instead reads + // beats from wherever the previous page stopped, and the tail is swept + // after the loop — so the day's beat window is covered exactly once. + var rrFrom = fromRecTs; while (true) { final decodedRows = await LocalDb.decodedOneHzBatchByRecTsRange( limit: _rawDecodeBatchSize, @@ -2659,17 +2737,17 @@ class DerivationEngine { rangePages: rangePages, rangeRows: rangeRows, ); - // The page is ordered rec_ts ASC, so first = min second, last = max. - // decoded_rr shares the rec_ts key, so this pulls exactly the page's - // beats — no counter span (which broke across the strap's reboot reset). - final firstRecTs = (decodedRows.first['rec_ts'] as num?)?.toInt(); + // The page is ordered rec_ts ASC, so last = max second. decoded_rr + // shares the rec_ts key, so [rrFrom, lastRecTs] is a PK range read — + // no counter span (which broke across the strap's reboot reset). final lastRecTs = (decodedRows.last['rec_ts'] as num?)?.toInt(); - final rrRows = firstRecTs == null || lastRecTs == null + final rrRows = lastRecTs == null ? const >[] : await LocalDb.decodedRrByRecTsRange( - fromRecTs: firstRecTs, + fromRecTs: rrFrom, toRecTs: lastRecTs, ); + if (lastRecTs != null) rrFrom = lastRecTs + 1; worker.send({'type': 'page', 'frames': decodedRows, 'rr': rrRows}); final last = decodedRows.last; afterRecTs = (last['rec_ts'] as num?)?.toInt() ?? afterRecTs; @@ -2679,6 +2757,23 @@ class DerivationEngine { } break; } + // The tail: beats after the last frame second (or, on a range with no + // frames at all, the whole range). Usually zero rows and one indexed + // lookup; when it is not, these are seconds the band recorded and only + // reported beats for. + if (rrFrom <= toRecTs) { + final tailRr = await LocalDb.decodedRrByRecTsRange( + fromRecTs: rrFrom, + toRecTs: toRecTs, + ); + if (tailRr.isNotEmpty) { + worker.send({ + 'type': 'page', + 'frames': const >[], + 'rr': tailRr, + }); + } + } worker.send(const {'type': 'finish'}); // BOUNDED. `result.future` had no timeout at all, so any path that left // the worker unable to answer hung this call — and with it the whole @@ -3634,6 +3729,10 @@ class DerivationEngine { // score and must carry its own tag; a day whose provenance is genuinely // unknown stays NULL and is never retro-filled with this. source: 'band', + // WHICH BAND'S UNITS these scalars are in (B5). Already null when the + // day's substrate spans two straps or carries no stamp — the same + // "unknown, never guessed" value the column is documented with. + deviceFamily: daySub.deviceFamily, series: { 'rhr': sc('rhr'), 'rmssd': sc('rmssd'), @@ -4652,7 +4751,13 @@ class DerivationEngine { samples.add(ana.HrSample(ts * 1000.0, s.hr[i].toDouble())); } final zoneSet = ana.HeartRateZones.zonesFromMaxHr(hrMax); - return ana.HeartRateZones.timeInZone(samples, zoneSet).toRoundedMinuteMap(); + // Null = the stream has no cadence `sampleCadenceSeconds` will vouch for. + // An empty map is already this function's "no zones" answer everywhere it + // is read; a zero-filled one would claim the day was measured and spent at + // rest. See `HeartRateZones.timeInZone`. + return ana.HeartRateZones.timeInZone(samples, zoneSet) + ?.toRoundedMinuteMap() ?? + const {}; } /// ONE HR-flex pass, returning the day's active, basal and total figures @@ -5173,7 +5278,11 @@ class DerivationEngine { scMap, liveStepsReal, liveStepsFromStrap: liveStepsFromStrap, - bandSteps: hardwareStepsFromCounter(daySub), + bandSteps: hardwareStepsFromCounter( + daySub, + cumulativeCounterModulus: + ana.calibrationFor(_stepCounterModulus, daySub.deviceFamily), + ), ); if (daySub.length < 60) return; @@ -5986,9 +6095,19 @@ class DerivationEngine { /// pretending to separate postures. Listed per family because the cut is only /// meaningful against that family's accel scale; an unknown strap gets no cut /// and the whole block refuses. - static const Map _quietEnmoCutG = { - ana.DeviceFamily.gen4: 0.02, - ana.DeviceFamily.gen5: 0.02, + /// On-chip step counters whose BEHAVIOUR we have established, by the family + /// stamped on the row: the modulus it wraps at, and — the part that actually + /// matters — that it is cumulative and does NOT reset at midnight. Both are + /// verified for gen5 (see `Substrate.stepCount`). A stamp that is not a key + /// here, including no stamp at all, gets no band-step number: summing deltas + /// off a resetting counter loses the day's whole pre-sync prefix, and it + /// cannot be told apart from a wrap afterwards. See + /// [hardwareStepsFromCounter]. + static const Map _stepCounterModulus = {'gen5': 65536}; + + static const Map _quietEnmoCutG = { + 'gen4': 0.02, + 'gen5': 0.02, }; @visibleForTesting diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index ebc0b0fa..a21d9af3 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -249,11 +249,13 @@ void derivationPrepareWorker(SendPort mainSendPort) { .whereType() .map((e) => e.cast()) .toList(); - if (frames.isNotEmpty) { - final rr = ((message['rr'] as List?) ?? const []) - .whereType() - .map((e) => e.cast()) - .toList(); + final rr = ((message['rr'] as List?) ?? const []) + .whereType() + .map((e) => e.cast()) + .toList(); + // A page with beats but NO frames is a real page, not an empty one — see + // addDecodedPage. Only a page with neither is a raw-hex page. + if (frames.isNotEmpty || rr.isNotEmpty) { state.addDecodedPage(frames, rr); return; } @@ -312,6 +314,15 @@ void derivationPrepareWorker(SendPort mainSendPort) { }); } +/// One decoded page + its beats through the accumulator, the way the worker +/// feeds it. Exists so the page seam — the `beat_ts_ms` coalesce and the +/// frame/beat union — is testable without spinning an isolate. +@visibleForTesting +Substrate substrateFromDecodedPage( + List> frames, + List> rrRows, +) => (_PrepareAccumulator()..addDecodedPage(frames, rrRows)).buildSubstrate(); + @visibleForTesting PreparedDerivationPayload prepareDerivationPayload( Substrate sub, { @@ -419,6 +430,24 @@ class _PrepareAccumulator { /// path) or more than one (the athlete changed straps inside this window) ⇒ /// null, i.e. UNKNOWN, and per-family metrics refuse. Cheaper than a /// per-second array and it answers the only question anyone asks. + /// + /// WHAT THIS DOES NOT CATCH, and cannot today: a change of UNIT within one + /// family. A warranty-replacement WHOOP 4 stamps `gen4` exactly like the one + /// it replaced, so the singleton test passes and one thermistor calibration + /// is applied across two physical units' ADC counts — `skin_temp_raw` then + /// gets z-scored straight across the seam. Nothing on a `decoded_onehz` row + /// distinguishes two units: `counter` is flash-local and resets on reboot, + /// and the one real per-unit identifier the app holds (the HELLO serial, via + /// `PairedDevice.serial`) lives in SharedPreferences, not on the row and not + /// in this isolate. + /// + /// ponytail: family-level guard only; the unit-level guard needs the + /// per-row `device_id` from the phase-2 storage re-key (BANDAGNOSTIC B1/B2), + /// after which this set keys on `device_id` and a second unit refuses the + /// same way a second family does. Deliberately NOT approximated here — the + /// alternatives (refusing every family, or guessing a unit change from a + /// data gap) cost every user their skin-temp driver on every day, to catch + /// an event that happens at most a handful of times in a band's life. final Set _families = {}; void _noteFamily(Object? v) { @@ -464,7 +493,7 @@ class _PrepareAccumulator { List> frames, List> rrRows, ) { - if (frames.isEmpty) return; + if (frames.isEmpty && rrRows.isEmpty) return; // Associate beats to frames by rec_ts (their shared key). The strap's counter // resets on reboot, so grouping by counter mis-joined two seconds that reused // one counter within a page. @@ -474,12 +503,40 @@ class _PrepareAccumulator { if (recTs == null) continue; rrByRecTs.putIfAbsent(recTs, () => >[]).add(row); } + // THE UNION OF FRAME SECONDS AND BEAT SECONDS, not just the frames. + // + // A beat can exist for a second that has NO `decoded_onehz` row, and the + // band produces those on purpose: gen4 historical R10-lite is hr-only, so + // `_queueDecodedOneHz` (db.dart) keeps it out of the 1 Hz store and banks + // its R-R block on its own. Walking only `frames` dropped every one of + // those beats on the floor — silently, and permanently once the record was + // acked and trimmed. The beat-only second still gets a 1 Hz slot so the + // positional arrays stay 1:1 with `tsSec`; its sensor channels land on the + // ABSENT sentinels (accel exactly (0,0,0), ADC 0, hr 0), never on a + // fabricated reading. + final frameByRecTs = >{}; + final seconds = []; for (final row in frames) { final recTs = _num(row['rec_ts'])?.toInt(); if (recTs == null || recTs <= 0) continue; - _noteFamily(row['device_family']); + if (frameByRecTs.containsKey(recTs)) continue; + frameByRecTs[recTs] = row; + seconds.add(recTs); + } + for (final recTs in rrByRecTs.keys) { + if (recTs > 0 && !frameByRecTs.containsKey(recTs)) seconds.add(recTs); + } + // Cheap when nothing was added (the page arrived sorted); required when it + // was, since every downstream slice assumes `tsSec` is ascending. + seconds.sort(); + for (final recTs in seconds) { + final row = frameByRecTs[recTs]; + _noteFamily(row?['device_family']); tsSec.add(recTs); - hr.add(plausibleHrOrZero(_num(row['hr'])?.toInt() ?? 0)); + // NULL (absent), a beat-only second (no row at all) and an impossible + // byte all land on the same 0 — the array's one "no usable HR this + // second" value. It is NOT a wear verdict; see [Substrate.hr]. + hr.add(plausibleHrOrNull(_num(row?['hr'])?.toInt() ?? 0) ?? 0); // The 1 Hz arrays are POSITIONAL — one entry per second, 1:1 with tsSec — // so a NULL sensor column (schema v39: absent, never coerced to a real // reading) must still occupy its slot. It lands as the array's ABSENT @@ -488,11 +545,11 @@ class _PrepareAccumulator { // (no real sensor reads 0 g on all three axes — see that doc) // ADC → 0, read back through the `v > 0` gate every ADC consumer uses // Same discipline as `stepCount`'s -1 below. - ax.add(_num(row['ax'])?.toDouble() ?? 0); - ay.add(_num(row['ay'])?.toDouble() ?? 0); - az.add(_num(row['az'])?.toDouble() ?? 0); - spo2Red.add(_num(row['spo2_red_raw'])?.toInt() ?? 0); - spo2Ir.add(_num(row['spo2_ir_raw'])?.toInt() ?? 0); + ax.add(_num(row?['ax'])?.toDouble() ?? 0); + ay.add(_num(row?['ay'])?.toDouble() ?? 0); + az.add(_num(row?['az'])?.toDouble() ?? 0); + spo2Red.add(_num(row?['spo2_red_raw'])?.toInt() ?? 0); + spo2Ir.add(_num(row?['spo2_ir_raw'])?.toInt() ?? 0); // gen4 stores skin temp as a raw ADC count (`skin_temp_raw`); gen5 v18 // decodes it to °C and stores THAT (`skin_temp_c`), leaving skin_temp_raw // NULL on every row. Reading only the gen4 column meant a WHOLE gen5 @@ -502,8 +559,8 @@ class _PrepareAccumulator { // carry whichever the row has, in centi-°C to stay integral and positive // through the `v > 0` gate every ADC consumer uses. A user who swaps // gen4 → gen5 mid-history steps the baseline once; the z re-settles. - final tempRaw = _num(row['skin_temp_raw'])?.toInt(); - final tempC = _num(row['skin_temp_c'])?.toDouble(); + final tempRaw = _num(row?['skin_temp_raw'])?.toInt(); + final tempC = _num(row?['skin_temp_c'])?.toDouble(); skinTemp.add(tempRaw ?? (tempC == null ? 0 : (tempC * 100).round())); // NO skin contact on the decoded path, and no column to read: the byte the // name refers to is the sign+exponent half of a float32, never a contact @@ -512,22 +569,39 @@ class _PrepareAccumulator { // `decoded_onehz.step_count` is NULL on every gen4 row and on any gen5 // row decoded before schema v34. NULL is ABSENT (-1), not 0: 0 is a real // reading from a band that has not moved since its counter last wrapped. - stepCount.add(_num(row['step_count'])?.toInt() ?? -1); + stepCount.add(_num(row?['step_count'])?.toInt() ?? -1); // CV-04a — the band's own "this second's beat is valid" flag, the first // of the five write-only gen5 columns to reach Substrate. // // NULL IS ABSENT (-1), NOT FALSE. gen4's R24 has no such field, so every // gen4 row is NULL here, and a `?? 0` would tell every downstream reader // that a whole generation's beats were rejected BY THE BAND. 0 is a real - // reading and only gen5 can produce it. `Substrate.hrValidAt` gates the - // read on `device_family == 'gen5'` on top of this. - hrValid.add(_num(row['hr_valid'])?.toInt() ?? -1); + // reading and only a decoder that read the flag off the wire can produce + // it. `Substrate.hrValidAt` gates the read on THIS sentinel alone — the + // `device_family == 'gen5'` check that used to sit on top of it was a + // band id in the neutral layer (BANDAGNOSTIC C12). + hrValid.add(_num(row?['hr_valid'])?.toInt() ?? -1); final beats = rrByRecTs[recTs]; if (beats == null) continue; for (final beat in beats) { - final rr = _num(beat['rr_ms'])?.toDouble(); - if (rr == null || rr <= 0) continue; - rrTsMs.add(_num(beat['rr_ts_ms'])?.toDouble() ?? recTs * 1000.0); + // The SAME physiological bound the raw-replay path applies + // (`decodeSubstrate`), not just `> 0`: an interval outside + // kMinPlausibleRrMs..kMaxPlausibleRrMs is a missed or doubled beat + // detection, and it has to be refused on both ingest paths or the two + // disagree about the same band's beats. + final raw = _num(beat['rr_ms']); + final rr = raw == null ? null : plausibleRrOrNull(raw); + if (rr == null) continue; + // `beat_ts_ms` FIRST — it is where the beat actually was. `rr_ts_ms` is + // `rec_ts * 1000`, so every beat in a record shares one millisecond and + // the axis is a whole-second staircase: an inter-record gap shorter than + // a second is invisible to `_beatTimes`' re-anchor test and gets spliced + // out of the time base. COALESCE, never fabricate — the column is NULL + // for every row banked before it existed and for any source with no + // sub-second, and there the staircase is still the honest best answer. + rrTsMs.add(_num(beat['beat_ts_ms'])?.toDouble() ?? + _num(beat['rr_ts_ms'])?.toDouble() ?? + recTs * 1000.0); rrMs.add(rr); } } diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 86522c78..ebe1a912 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -315,6 +315,10 @@ Map deriveDayBundle(Map inputJson) { // ── HR over the SLEEP WINDOW (RHR / dip night-side) ──────────────────────── final sleepHr = [for (final h in d.sleepHr) h.toDouble()]; + // The clock for that series — parallel by construction (both are `Substrate` + // columns, 1:1 with `tsSec`). Seconds as doubles because that is what the + // analytics window helpers take. + final sleepTs = [for (final t in d.sleepTsSec) t.toDouble()]; // ── RR over the SLEEP WINDOW → cleaned NN → HRV (the V2 fix) ─────────────── // HRV/RHR are rest/sleep-only per the catalog. Running correctRr+hrvTime over @@ -384,14 +388,22 @@ Map deriveDayBundle(Map inputJson) { // heart rate at any tier — it is a different quantity wearing the label. The // only honest output is absence, so the card says why instead. // - // `sleepHr` must be a POSITIONALLY DENSE 1 Hz series where 0 means off-skin — - // `nocturnalRhr` slides its 30-minute window over wall-clock POSITIONS and - // enforces a minimum on-skin coverage per window. Passing a compacted series - // defeats that: with gaps squeezed out, 1800 consecutive entries could span - // many hours, so "lowest 30-minute mean" silently becomes "lowest mean over - // whatever 1800 samples happened to survive". + // THE 30 MINUTES ARE NOW 30 MINUTES. `nocturnalRhr` used to slide a window of + // 1800 POSITIONS, which is half an hour only where every position is exactly + // one second — so the series had to be positionally dense at 1 Hz or "the + // lowest 30-minute mean" quietly became "the lowest mean over whatever 1800 + // samples survived". It is a real condition, not a hypothetical: the owner's + // own export has sleep windows missing 506 seconds of 25,262, and a 15 s band + // publishes a measured 59.7 bpm night as 66.4. + // + // Passing `sleepTs` moves the window onto the WALL CLOCK, so density stops + // being a precondition: 30 min is 30 min at any cadence, off-skin gaps are + // still never compacted away (a window must carry `minCoverage` of the + // samples it SHOULD hold at the stream's own measured cadence), and a stream + // with no measurable cadence yields ABSENCE rather than a guess. The + // no-sleep-no-RHR branch above is untouched by any of that. final rhr = (hasSleep && sleepHr.isNotEmpty) - ? nocturnalRhr(sleepHr) + ? nocturnalRhr(sleepHr, tsSec: sleepTs) : const Metric.absent( tier: Tier.high, inputs_used: ['hr_1hz', 'sleep_window'], @@ -1554,7 +1566,10 @@ Map _wakeZoneMinutesFromSeries( final samples = [ for (final p in wakeHr) HrSample(p.tsSec * 1000.0, p.hr), ]; - return HeartRateZones.timeInZone(samples, zoneSet).toRoundedMinuteMap(); + // Null = no cadence `sampleCadenceSeconds` will vouch for; `const {}` is the + // caller's existing absent state (see `hrZones` at its declaration). + return HeartRateZones.timeInZone(samples, zoneSet)?.toRoundedMinuteMap() ?? + const {}; } List> _zoneTimeline( diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 032de077..431a4279 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -29,21 +29,135 @@ const double kMinAccelCoverageForVanHees = 0.5; /// Physiological bound on a 1 Hz heart rate (bpm), inclusive. /// -/// The gen4 TRUSTED decode path (v24 / v12) returns the HR byte verbatim with -/// no bound — the protocol's `_physiologicallyPlausible` gate runs only on the -/// best-effort versions, and gen5 v18 bounds it independently — so one -/// corrupt-but-CRC-valid byte of 250 used to pass the `hr > 0` filter and land -/// straight in the day's max HR. Applying this in the protocol would cost the -/// WHOLE record (accel and RR with it); applied here it costs only the second. +/// HUMAN PHYSIOLOGY, NOT A SENSOR PROPERTY, so it is the same number on every +/// band: no heart beats 24 times a minute or 231 times a minute for a whole +/// second. Nothing about the strap moves it, which is why it does not go +/// through `calibrationFor` and why an unstamped record still gets it. +/// +/// It exists because the gen4 TRUSTED decode path (v24 / v12) returns the HR +/// byte verbatim with no bound — the protocol's `_physiologicallyPlausible` +/// gate runs only on the best-effort versions, and gen5 v18 bounds it +/// independently — so one corrupt-but-CRC-valid byte of 250 used to pass the +/// `hr > 0` filter and land straight in the day's max HR. Applying it in the +/// protocol would cost the WHOLE record (accel and RR with it); applied here it +/// costs only the second. const int kMinPlausibleHr = 25; const int kMaxPlausibleHr = 230; -/// The HR to store for second-level [raw]: itself when physiologically -/// possible, else `0` — the substrate's existing "no usable HR this second" -/// value, which every reader already filters on. Never clamped to the bound: -/// a corrupt byte must not become a plausible reading. -int plausibleHrOrZero(int raw) => - (raw >= kMinPlausibleHr && raw <= kMaxPlausibleHr) ? raw : 0; +/// [raw] when it is a heart rate a human can have, else NULL — the reading is +/// refused, never clamped to the bound (a corrupt byte must not become a +/// plausible reading) and never reported as a measurement of anything else. +/// +/// Callers that must land it in the dense [Substrate.hr] array write +/// `plausibleHrOrNull(raw) ?? 0`, 0 being that array's ONE "no usable HR this +/// second" value — see [Substrate.hr] for why that is not a claim about wear. +int? plausibleHrOrNull(int raw) => + (raw >= kMinPlausibleHr && raw <= kMaxPlausibleHr) ? raw : null; + +/// Physiological bound on one R-R interval (ms), inclusive — the [kMinPlausibleHr] +/// / [kMaxPlausibleHr] window expressed as the gap between two beats, widened +/// at the long end because a single interval is not a rate: one dropped beat +/// doubles the gap without the heart doing anything unusual, and 2,400 ms still +/// sits inside the range an ordinary Malik/Lipponen ectopic filter is built to +/// see and correct. Below 250 ms (240 bpm sustained across one beat) it is not +/// a beat detection. +const int kMinPlausibleRrMs = 250; +const int kMaxPlausibleRrMs = 2400; + +/// [rrMs] when it is an interval a human heart can produce, else NULL. +double? plausibleRrOrNull(num rrMs) => + (rrMs >= kMinPlausibleRrMs && rrMs <= kMaxPlausibleRrMs) + ? rrMs.toDouble() + : null; + +/// The largest acceleration MAGNITUDE (g) a wrist can hold for a WHOLE SECOND. +/// +/// These samples are one-second gravity vectors, not raw 100 Hz: an impact, a +/// swing and a free-fall are all sub-second transients that average away before +/// they get here, so this bounds a SUSTAINED magnitude, not a peak. Holding 4 g +/// for a full second is ~3 g of net force for a second — aerobatics and +/// centrifuges, not anything a band is worn through. +/// +/// It is deliberately NOT the part's full-scale range: an FSR is an encoding +/// choice, and a part configured to +/-2 g with a scale-error decoder sails +/// through a +/-16 g test. It is also deliberately far above protocol's +/// `_physiologicallyPlausible` gravity window (magSq 0.25..3.24, i.e. 0.5..1.8 g), +/// which assumes a low-pass-filtered gravity vector and had to be dropped from +/// the gen5 decoder in 539a97b because it rejected real workout seconds. There +/// is no lower bound here for the same reason: sustained low-g seconds are real +/// and only EXACT zero is evidence of a fill. +const double kMaxSustainedAccelG = 4.0; + +/// Whether a 1 Hz gravity triplet is a MEASUREMENT. +/// +/// Two rejections, both physical: +/// * exact `(0, 0, 0)` — no accelerometer reads zero on all three axes at +/// rest or in motion, so it is an all-zero payload (see +/// [Substrate.accelPresentAt] for what that costs if it is trusted). +/// * a magnitude above [kMaxSustainedAccelG] — see there. +bool accelPlausible(double ax, double ay, double az) { + final magSq = ax * ax + ay * ay + az * az; + return magSq > 0 && magSq <= kMaxSustainedAccelG * kMaxSustainedAccelG; +} + +/// Where each beat in one record actually sits, in absolute epoch ms — or +/// null for a beat that cannot be placed. One entry per entry in [rrMs]. +/// +/// TWO PARTS, AND THEY ARE NOT EQUALLY SOLID. Read them separately. +/// +/// THE ANCHOR IS MEASURED. `rec_ts + tsSubsec/32768` is the record's own +/// timestamp, whole seconds and sub-second, exactly as the strap sent it. +/// This app has dropped the second half of that since forever, pinning every +/// record to a whole second. With no sub-second there is no anchor and every +/// beat here is null; a whole second is NOT substituted for one, because the +/// whole point of this column is to say something the old one could not. +/// +/// THE PLACEMENT IS A MODEL, and it is one assumption wide: an R-R interval +/// is the gap ENDING at its beat (that part is the definition), and the LAST +/// beat a record reports sits at the record's timestamp. Everything else +/// follows — beat i is the anchor minus the intervals after it. The direction +/// is chosen because backwards is the only one that cannot place a beat in +/// the future, i.e. after the moment we were told about it; a forward walk +/// would also run every multi-beat record past its own second (the intervals +/// sum to 1,426 ms on a 2-beat record and 2,594 ms on a 4-beat one, measured) +/// and straight through the next record's. +/// +/// WHAT THE INTERVALS DO NOT DO IS TILE THE SECOND. Over 81 uninterrupted +/// runs of 300+ consecutive records in a real export, the intervals sum to +/// 0.967 of the `rec_ts` span (0.960-0.990 across runs) — so the beat train is +/// a CHAIN that runs a few percent short, which is what a handful of rejected +/// beats looks like, and not a set of per-second buckets. Several of a +/// record's intervals reach back out of its own second. Per-record placement +/// is nevertheless what THIS path can do — records arrive batched, out of +/// order and with gaps, so no cross-record chain is available at the write — +/// and a consumer that wants the chain can walk `rr_ms` itself. +/// +/// WHAT MOVES AND WHAT DOES NOT, now that `Substrate.rrTsMs` is fed from here +/// rather than from the `rec_ts * 1000` staircase. The INTERVAL SERIES does +/// not move at all — same values, same order — so `hrvTime(nn)` with no time +/// axis is bit-identical, and so is SDNN, which never pairs. Everything that +/// takes the axis does move: a Lomb-Scargle periodogram handed beats where +/// they happened, a beat put on the same axis as a motion sample, a real +/// inter-record gap. That includes RMSSD and pNNx as production calls them +/// (`hrvTime(nn, nnTimesMs: …)`), because the axis decides which successive +/// pairs count as CONTIGUOUS — measured at +0.03% RMSSD / +0.6% pNN50 over a +/// real 6 h block of 27,114 beats. +/// +/// A non-positive interval BREAKS THE CHAIN: the gap before that beat is +/// unknown, so every EARLIER beat in the record becomes unplaceable and gets +/// null rather than a position computed as if the missing gap were zero. +List beatTimesMs(int recTs, int? tsSubsec, List rrMs) { + final out = List.filled(rrMs.length, null); + if (tsSubsec == null || rrMs.isEmpty) return out; + final anchor = recTs * 1000 + (tsSubsec * 1000) ~/ 32768; + var back = 0; + for (var i = rrMs.length - 1; i >= 0; i--) { + out[i] = anchor - back; + if (rrMs[i] <= 0) break; + back += rrMs[i]; + } + return out; +} /// The decoded 1 Hz substrate — the only decoded form (ARCHITECTURE_V2). /// @@ -54,7 +168,23 @@ class Substrate { /// Epoch seconds, 1 Hz, sorted ascending. One entry per R24 record. final List tsSec; - /// 1 Hz HR (bpm). 0 = off-skin (never bradycardia). Parallel to [tsSec]. + /// 1 Hz HR (bpm). Parallel to [tsSec]. **`0` means NO USABLE HEART RATE this + /// second — it is not a claim that the band was off your wrist.** + /// + /// The doc here used to say "0 = off-skin", and that was a second assertion + /// smuggled in beside the first. Three different facts land on this 0: the + /// record carried no HR field, the sensor found no beat, and the byte was + /// outside [kMinPlausibleHr]..[kMaxPlausibleHr]. Read as "off-skin" the last + /// of those censors a reading AND replaces it with a wear verdict, which + /// biases anything that counts on-skin seconds — nocturnal RHR most of all, + /// on exactly the calm nights that push HR toward the low bound. + /// + /// It is one value rather than two because the ledger has already collapsed + /// them: `decoded_onehz.hr` stores NULL for every record with no heart rate + /// (db.dart `_queueDecodedOneHz`, `decoded.hr > 0 ? decoded.hr : null`), so + /// "the band said zero" cannot reach this array as anything else. Every + /// reader gates `> 0`. Wear truth lives in the HELLO body, the wrist on/off + /// events and the record-presence runs (`_wearBlock`) — never here. final List hr; /// Beat-to-beat RR: interval end time (epoch ms) + interval (ms). Sparse. @@ -98,18 +228,19 @@ class Substrate { /// no such field, so every gen4 second reads -1, and reading that as `false` /// would turn "this band cannot say" into "the band said no". /// - /// GEN5 ONLY, and gated twice on purpose: the sentinel above, and - /// [deviceFamily]. A gen4 strap and a gen5 strap will tier DIFFERENTLY on - /// identical physiology because of exactly this kind of extra evidence, so a - /// reader that weights by it has to say so somewhere the user can see. + /// GATED ON THE SENTINEL ALONE — see [hrValidAt] for why the band-id check + /// that used to sit beside it is gone. A strap that reports this flag and a + /// strap that cannot will tier DIFFERENTLY on identical physiology because of + /// exactly this kind of extra evidence, so a reader that weights by it has to + /// say so somewhere the user can see. /// Same absent-marker discipline as [stepCount] and [accelPresentAt]. final List hrValid; /// WHICH STRAP MEASURED THIS SUBSTRATE — `'gen4'`, `'gen5'`, or null. /// /// Stamped at ingest into `decoded_onehz.device_family` and carried here so - /// the pure pipeline can dispatch on it (analytics: `deviceFamilyOf` → - /// `calibrationFor`). It is ONE value for the whole substrate, not a + /// the pure pipeline can dispatch on it (analytics: `calibrationFor`, whose + /// map of families is open — a stamp is a key or it is not). It is ONE value for the whole substrate, not a /// per-second array, because the question a metric asks is "which sensor /// package produced this window", and a window that mixes two answers has no /// single answer. @@ -233,6 +364,11 @@ class Substrate { /// EXACTLY 0.0 is an all-zero payload, i.e. no measurement. Exact zero is /// therefore the ABSENT marker. /// + /// The upper end is [kMaxSustainedAccelG] — a second-long mean no wrist + /// holds — so a decoder whose scale factor is wrong by an order of magnitude + /// stops reading as violent movement and starts reading as absent. There is + /// no lower bound beyond exact zero; see [accelPlausible]. + /// /// This used to rest on "every decoder gates on `magSq >= 0.25`", which is no /// longer true — protocol 539a97b dropped that gate from the gen5 v18 decoder /// (it is a bound on a NORMALISED gravity vector and gen5 emits per-axis raw @@ -247,7 +383,7 @@ class Substrate { /// rule reads as PERFECT IMMOBILITY. Eight hours of missing accel scores /// 28 501 immobile seconds and yields a fabricated ~7.9 h sleep window, /// fully staged. Absent input must produce no claim, never a confident one. - bool accelPresentAt(int i) => !(ax[i] == 0 && ay[i] == 0 && az[i] == 0); + bool accelPresentAt(int i) => accelPlausible(ax[i], ay[i], az[i]); /// Fraction of [lo, hi) seconds carrying a real gravity vector (0..1). /// Returns 0 for an empty range — no evidence, not "all present". @@ -280,9 +416,16 @@ class Substrate { /// ABSENT, NEVER FALSE. A gen4 strap has no such field and a NULL read as /// `false` would silently mark a whole generation's beats untrustworthy. bool? hrValidAt(int i) { - // Unknown provenance refuses outright: the column is gen5's, and a row with - // no device stamp cannot be shown to have come from one. - if (deviceFamily != 'gen5') return null; + // THE ROW ANSWERS FOR ITSELF. This used to also require + // `deviceFamily == 'gen5'` — a band id hardcoded inside the class whose + // whole job is to be neutral, which meant a second band that reports the + // same flag would have been silently ignored while its data sat right here. + // The sentinel is the evidence: only a decoder that read the flag off the + // wire writes a non-negative value into this column (db.dart writes NULL + // otherwise, and derive_prepare lands NULL on -1), so a value >= 0 IS a + // declaration by the source that produced the row. An unstamped substrate + // carrying real flags is a source that told us the flag and not the badge — + // refusing it discards a measurement to punish missing metadata. if (i < 0 || i >= hrValid.length) return null; final v = hrValid[i]; return v < 0 ? null : v != 0; @@ -514,7 +657,8 @@ Substrate decodeSubstrate(List hexes) { final live = proto.realtimeRr(hex); if (live != null && live.ts > 0) { for (final v in live.rrMs) { - if (v > 0) looseRr.add(_Beat(live.ts * 1000.0, v.toDouble())); + final rr = plausibleRrOrNull(v); + if (rr != null) looseRr.add(_Beat(live.ts * 1000.0, rr)); } } } @@ -535,7 +679,7 @@ Substrate decodeSubstrate(List hexes) { for (var i = 0; i < n; i++) { final r = recs[i].r; tsSec[i] = r.tsEpoch; - hr[i] = plausibleHrOrZero(r.hr); + hr[i] = plausibleHrOrNull(r.hr) ?? 0; if (r.accelG.length == 3) { ax[i] = r.accelG[0]; ay[i] = r.accelG[1]; @@ -551,13 +695,24 @@ Substrate decodeSubstrate(List hexes) { skinTemp[i] = r.skinTempRaw; // ignore: deprecated_member_use skinContact[i] = r.skinContact; - // RR beats: anchored at the record second (epoch ms). Beats within a record - // share its second; time order is preserved by the record sort above. + // RR beats: placed at their MEASURED instant (`beatTimesMs` — the record's + // own sub-second anchor, intervals walked backwards from it), falling back + // to the record second only when the record carries no sub-second. Beats + // used to all share the record's whole second here, which says two beats + // 800 ms apart happened at the same millisecond. Emission ORDER is + // unchanged (record order, then beat order) so no interval series moves — + // only where the beats sit on the clock. final t = r.tsEpoch * 1000.0; - for (final rr in r.rrIntervalsMs) { - if (rr > 0) { - rrMs.add(rr.toDouble()); - rrTsMs.add(t); + final beatTs = beatTimesMs(r.tsEpoch, r.tsSubsec, r.rrIntervalsMs); + for (var b = 0; b < r.rrIntervalsMs.length; b++) { + // A non-positive interval was already dropped here; the bound only widens + // that to intervals no heart produces. It drops the BEAT, not the record: + // `beatTimesMs` has already placed the survivors, and an interval this + // far out is a missed or doubled detection, not a rhythm. + final rr = plausibleRrOrNull(r.rrIntervalsMs[b]); + if (rr != null) { + rrMs.add(rr); + rrTsMs.add(beatTs[b]?.toDouble() ?? t); } } } @@ -607,9 +762,22 @@ Substrate decodeSubstrate(List hexes) { /// since R24 has no pedometer field. Null means "this hardware cannot count /// steps", never "you took no steps". /// -/// `stepMotionCounter` is a **cumulative u16** that wraps at 65536 and is also -/// reset by a strap reboot/re-pair, so the day's total is the sum of positive -/// per-record deltas, not `last - first`. Two hazards, both handled here: +/// [cumulativeCounterModulus] IS A DECLARATION AND IT IS NOT OPTIONAL — +/// null abstains. It says two things about the source's counter, and this +/// function is only correct if both hold: it wraps at that modulus, and it is +/// CUMULATIVE, i.e. it does not reset inside the window being summed. Both used +/// to be assumed (`wrap = 65536`, hardcoded), and the second one is the +/// expensive assumption: this reads DELTAS, so a counter that resets at +/// midnight silently loses every step taken before the day's first synced +/// record — 12,500 walked, 8,300 published, at tier HIGH and confidence 0.9, +/// with nothing in the output saying so. Nothing on a record distinguishes a +/// reset from a wrap after the fact, so an undeclared counter gets no number +/// rather than a guessed one. The caller declares it because the caller knows +/// which band stamped the rows; see `DerivationEngine._stepCounterModulus`. +/// +/// The counter is also reset by a strap reboot/re-pair, so the total is the sum +/// of positive per-record deltas, not `last - first`. Two hazards, both handled +/// here (`wrap` below is [cumulativeCounterModulus]): /// /// * **wrap** (65500 → 100): the raw delta is negative. Re-reading it modulo /// 65536 gives the true small delta, which passes the plausibility budget. @@ -633,8 +801,13 @@ Substrate decodeSubstrate(List hexes) { /// /// A delta is either credited in full or dropped in full, so this function can /// never return a negative or an absurd total, whatever the counter does. -int? hardwareStepsFromCounter(Substrate sub, {int maxStepsPerSecond = 5}) { - const wrap = 65536; +int? hardwareStepsFromCounter( + Substrate sub, { + required int? cumulativeCounterModulus, + int maxStepsPerSecond = 5, +}) { + final wrap = cumulativeCounterModulus; + if (wrap == null || wrap <= 0) return null; const minGapSecForBudget = 60; const maxGapSecForBudget = 3600; int? prev; diff --git a/lib/data/db.dart b/lib/data/db.dart index 31c7f997..0a49146b 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -19,6 +19,8 @@ import 'package:openstrap_protocol/openstrap_protocol.dart' as proto; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; import 'package:sqflite/sqflite.dart'; + +import '../compute/substrate.dart' show beatTimesMs; // The ONE thing this layer takes from compute/: the running build's algo // version, which every day_result read applies as a CEILING (see [dayResult]). // `show` keeps the rest of the engine out of this namespace. @@ -31,6 +33,7 @@ import 'live_coverage_policy.dart'; import 'med_store.dart'; import 'models.dart'; import 'nutrition_store.dart'; +import 'observation.dart'; import 'series_codec.dart'; /// The outcome of a database rebuild: why the old file would not open, where it @@ -250,7 +253,7 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 46; + static const int schemaVersion = 48; /// SQLite caps host parameters per statement (`SQLITE_MAX_VARIABLE_NUMBER` — /// only 999 on the builds shipped with older Android/iOS). Any `IN (?, ?, …)` @@ -258,6 +261,20 @@ class LocalDb { /// `decoded_onehz` is 86 400 counters. Same reason `commitSyncBatch` chunks. static const int _maxSqlVars = 500; + /// THE PRIMARY BAND'S `device_id`, and it is reserved permanently. + /// + /// Not a migration default — a standing rule, load-bearing twice over: + /// * The newest-wins dedupe on `decoded_onehz` only works while every row + /// from the one physical band shares one key value, so a post-reboot + /// counter reset and a re-drained record still collide. + /// * A BLE `remoteId` is NOT a stable identity (per-app CBPeripheral UUID + /// on iOS, a rotating RPA on Android). Letting one reach the key would + /// fragment one band into N identities across reinstalls. + /// + /// A SECONDARY device gets a real id, issued by its adapter from something + /// the band emits across the handshake — never from the link. + static const String kPrimaryDeviceId = ''; + /// Split [items] into `_maxSqlVars`-sized chunks for `IN (…)` binding. static Iterable> _sqlVarChunks(List items) sync* { for (var i = 0; i < items.length; i += _maxSqlVars) { @@ -774,6 +791,33 @@ class LocalDb { // _retireDisprovenOneHzColumns for the evidence. await _retireDisprovenOneHzColumns(db); } + if (oldV < 47) { + // Put `device_id` in front of the key of the three stores whose + // identity was a WHOOP-shaped quantity (a record second, the band's + // flash counter). The one item on the band-agnostic list that cannot + // be done after a second device has written — see + // [_rekeyStoresByDeviceId]. Rewrites no value and moves no derived + // number, so it ships without a kAlgoVersion bump. + // + // LAST on the ladder, after every ADD COLUMN above, because the + // rebuild reconstructs its DDL from `PRAGMA table_info` and so + // carries across exactly the columns that exist when it runs. + await _rekeyStoresByDeviceId(db); + } + if (oldV < 48) { + // The observation store — vendor-computed, typed-in and imported + // scalars. CREATE TABLE IF NOT EXISTS + two indexes and NOTHING + // else: no backfill, no rewrite, no ADD COLUMN, nothing read. It is + // a new empty table, so the CHECK and the UNIQUE index have no + // existing row to fail on — which matters, because a throw in here + // rolls the WHOLE ladder back and quarantines the user's database + // (invariant 11), and this rung runs after the v47 re-key that every + // upgrading install is already paying for on the launch path. + // + // Ships without a kAlgoVersion bump, deliberately: nothing writes + // this table, nothing reads it, and no derived number can move. + await _createObservation(db); + } }, onOpen: (db) async { await _repairOpenSchema(db); @@ -832,6 +876,7 @@ class LocalDb { // version number here would collide with the other branches reaching for // the next one, and buy nothing a no-op CREATE does not already do. await _createImportedWorkout(db); + await _createObservation(db); await _createCycleSymptom(db); await _ensureSessionSchema(db); await _ensureSyncStateSchema(db); @@ -990,7 +1035,7 @@ class LocalDb { /// reading share `decoded_onehz`'s columns, so a wrong stamp is worse than no /// stamp. Existing rows — and anything imported, which came from no link at /// all — read NULL, meaning UNKNOWN PROVENANCE, which readers must treat as - /// its own case (analytics: `deviceFamilyOf` → null → refuse) rather than as + /// its own case (analytics: `calibrationFor` → null → refuse) rather than as /// gen4. /// /// Nullable TEXT, no DEFAULT, ever: a default would turn "we don't know" into @@ -1435,6 +1480,117 @@ class LocalDb { ); } + // ── OBSERVATIONS (vendor-computed / typed-in / imported scalars) ─────────── + /// Everything that is NOT a raw sensor sample and NOT computed by us: + /// a `reports` band's own conclusions, the user's typed-in numbers, and + /// another app's history. See docs/OBSERVATION_SPEC.md. + /// + /// THE SINGLE INVARIANT, and it is the whole safety story: **nothing reads + /// this table into a baseline, into a trend that also carries derived + /// values, or into any input to a derivation.** Held the same way + /// `imported_measurement` holds its own version of the rule — a SEPARATE + /// table, so the only way to violate it is to name it somewhere new, and + /// `observation_isolation_test.dart` fails the moment anyone does. If it is + /// ever violated the failure is SILENT: an unexplained step change in a + /// long-horizon number, months later, with no way to tell which day broke it. + /// + /// IDENTITY IS AN EXPRESSION INDEX, NOT A PRIMARY KEY, and that is not a + /// stylistic choice. The spec's key is + /// `(device_id, ts_ms, source_kind, COALESCE(vendor_key, key))`, and SQLite + /// takes no expression in a table-level PRIMARY KEY. The two obvious repairs + /// both fail: + /// + /// * A `GENERATED ALWAYS AS (COALESCE(...)) STORED` column is rejected + /// outright — "generated columns cannot be part of the PRIMARY KEY" — and + /// generated columns need SQLite 3.31 (2020) besides, which minSdk 26 + /// (Android 8 ships 3.18) does not have. + /// * A plain composite `PRIMARY KEY (…, vendor_key, key)` COMPILES AND IS + /// WRONG. A PRIMARY KEY on a rowid table is a UNIQUE INDEX, and SQLite + /// treats NULLs in one as DISTINCT — so two rows with the same + /// `vendor_key` and a NULL `key` do not collide, `INSERT OR REPLACE` does + /// not replace, and a re-import silently doubles every composite it + /// carries. Proven, not assumed. + /// + /// A UNIQUE INDEX may hold an expression (SQLite 3.9, 2015 — safely below + /// the Android 8 floor), `INSERT OR REPLACE` resolves against it exactly as + /// it would against a PRIMARY KEY, and `COALESCE` is non-NULL whenever the + /// CHECK holds, so the NULL-distinctness trap never arms. The CHECK is what + /// makes that true, which is why it is a DB constraint and not a Dart assert. + /// + /// `device_id = ''` is the primary band, permanently — the standing rule + /// established for `decoded_onehz` at v47, and for the same reason: a BLE + /// remoteId is not a stable identity. + /// + /// ponytail: scalars only (`value REAL` + `unit TEXT`). A vendor hypnogram + /// or a vendor GPS track is not a scalar and has no consumer — give it a + /// table of its own when something is actually going to read it. + static Future _createObservation(Database db) async { + await db.execute(''' + CREATE TABLE IF NOT EXISTS observation ( + device_id TEXT NOT NULL DEFAULT '', + ts_ms INTEGER NOT NULL, + date TEXT NOT NULL, + source_kind TEXT NOT NULL, + vendor_key TEXT, + key TEXT, + value REAL, + unit TEXT, + attribution TEXT NOT NULL, + CHECK (vendor_key IS NOT NULL OR key IS NOT NULL) + ) + '''); + await db.execute( + 'CREATE UNIQUE INDEX IF NOT EXISTS idx_observation_identity ' + 'ON observation(device_id, ts_ms, source_kind, COALESCE(vendor_key, key))', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_observation_date ' + 'ON observation(date, source_kind)', + ); + } + + /// Upsert observations. Idempotent on `(device_id, ts_ms, source_kind, name)` + /// so re-running an import re-states rather than duplicates. + /// + /// `date` comes from [dayLabelOf] and nowhere else — the day model is LOCAL + /// calendar days, and a label computed any other way is wrong for 23 h/25 h + /// DST days and for every user whose local date differs from UTC's. + /// + /// There is deliberately NO reader here yet. Nothing writes observations + /// either: the adapter seam that will is phase 4, and wiring an existing + /// importer into this table today would move numbers users already see. + static Future putObservations( + List rows, { + String deviceId = kPrimaryDeviceId, + }) async { + if (rows.isEmpty) return 0; + final db = await instance; + await db.transaction((txn) async { + final batch = txn.batch(); + for (final o in rows) { + batch.insert('observation', { + 'device_id': deviceId, + 'ts_ms': o.at.millisecondsSinceEpoch, + 'date': o.date, + 'source_kind': o.sourceKind.name, + 'vendor_key': o.vendorKey, + 'key': o.key, + 'value': o.value.toDouble(), + 'unit': o.unit, + 'attribution': o.attribution, + }, conflictAlgorithm: ConflictAlgorithm.replace); + } + await batch.commit(noResult: true); + }); + return rows.length; + } + + /// One observation. [putObservations] is the batch this delegates to. + static Future putObservation( + Observation o, { + String deviceId = kPrimaryDeviceId, + }) => putObservations([o], deviceId: deviceId); + // ── IMPORTED WORKOUTS (Apple Health / Health Connect) ────────────────────── /// A workout some OTHER app recorded, held in its own table for the same /// reason `imported_measurement` is: so that it CANNOT become one of ours. @@ -1781,6 +1937,14 @@ class LocalDb { // Anything not explicitly 'phone' is band — pre-v27 rows default to // it, and relabelling them would suppress a real band count. fromBand: r['source'] != kStepSourcePhone, + // ponytail: every row is the primary band's until `live_coverage` + // carries a `device_id` of its own (the decoded tables got one at + // schema 47; this table did not). `CoverageSpan.deviceId` defaults to + // `''` and `resolveDaySteps` treats one id as one sensor, so this is + // exactly today's behaviour — the day a second strap can write here, + // select the column and pass it, and the equal-rank double-count is + // already handled. + ), ]); } @@ -2006,7 +2170,28 @@ class LocalDb { // service discovery). Null = the caller could not name it, which lands as // NULL = unknown provenance. Never defaulted to gen4. String? deviceFamily, + // WHICH PHYSICAL DEVICE these rows belong to (see [kPrimaryDeviceId]). + String deviceId = kPrimaryDeviceId, }) async { + // B4: `sync_cursor` is a SINGLE GLOBAL `name TEXT PRIMARY KEY` namespace + // shared with everything else that keeps a scalar (`frozen_headline`, …), + // and `band_backlog` carries `device_family` but no device id. So the trim + // token, `counter_hw`, `rec_ts_hw` and the data range of a SECOND + // offloading band would land on the first band's cursor and mis-trim its + // flash. Namespacing that key space is a phase-4 job (it needs the device + // table and a decision about which keys are per-device at all); refusing is + // the honest thing until then, and it refuses in the SAFE direction — + // nothing commits, so nothing is ACKed and the band keeps its data. + // + // ponytail: global cursor namespace, one offloading device. Per-device + // namespacing when a second adapter can actually offload. + if (deviceId != kPrimaryDeviceId) { + throw StateError( + 'commitSyncBatch: sync_cursor is a single global namespace and cannot ' + 'hold a second offloading device (deviceId="$deviceId"). Namespace it ' + 'before enabling a non-primary offload path.', + ); + } void checkpoint(String msg) { try { onCheckpoint?.call(msg); @@ -2119,6 +2304,11 @@ class LocalDb { final sample = samples[i]; if (sample != null) { batch.insert('samples', { + 'device_id': deviceId, + // From `ts`, not `recTs`: `toDbMap` writes `ts: sample.tsEpoch` + // and the migration derives `ts_ms` from `ts`, so the key and the + // column it is built from must not be allowed to disagree. + 'ts_ms': sample.tsEpoch * 1000, 'counter': raw.counter, ...sample.toDbMap(), }, conflictAlgorithm: ConflictAlgorithm.ignore); @@ -2129,6 +2319,7 @@ class LocalDb { raw, sample, deviceFamily: deviceFamily, + deviceId: deviceId, ); if (raw.counter > maxCounter) maxCounter = raw.counter; if (recTs > maxRecTs) maxRecTs = recTs; @@ -2261,6 +2452,28 @@ class LocalDb { } catch (_) { /* already present */ } + // WHICH BAND'S UNITS this day's scalars are in (B5). Same table, same + // write, same read as `source` — and the same rules: nullable, no DEFAULT, + // NEVER retro-filled, because a guessed provenance is worse than none. + // + // This is live TODAY on a gen4→gen5 swap, with no second device involved: + // `skin_temp_adc` holds gen4 ADC COUNTS on one side of the swap and gen5 + // CENTI-DEGREES on the other, under one `metric_series` key, feeding one + // 28-day baseline that every nightly z-score is taken against. The seam is + // a step change in the units, not in the person. + // + // NOT the same as [_importedDatesSql]'s question. Imported days are ANOTHER + // ALGORITHM'S OUTPUT and are masked out of every baseline. A foreign family + // is this app's own maths over a different sensor, so it is masked + // PER-METRIC — only where the seam makes the number WRONG (different + // units), never merely noisier. See [foreignFamilyDates]. + try { + await db.execute( + 'ALTER TABLE metric_series_version ADD COLUMN device_family TEXT', + ); + } catch (_) { + /* already present */ + } } /// Backfill the stamp from `day_result`, which has carried the version all @@ -3221,12 +3434,22 @@ class LocalDb { // samples — LEGACY header-only record index (counter, ts, hr). Retained only // so pre-v11 databases stay readable if decoded_onehz backfill was partial. // New writes should go to decoded_onehz instead. + // + // v47: re-keyed off `counter` for the same reason decoded_onehz was re-keyed + // off it at v33 and off rec_ts now — `counter` is a WHOOP FLASH RECORD + // COUNTER, so a second offloading band's counter 500 is a different reading + // that collides with the first band's. The insert is IGNORE, so the collision + // DROPPED the newer row rather than replacing the older one, and + // [latestSample]'s fallback then served a stale second. static Future _createSamples(Database db) async { await db.execute(''' CREATE TABLE IF NOT EXISTS samples ( - counter INTEGER PRIMARY KEY, + device_id TEXT NOT NULL DEFAULT '', + ts_ms INTEGER NOT NULL DEFAULT 0, + counter INTEGER, ts INTEGER NOT NULL, - hr INTEGER + hr INTEGER, + PRIMARY KEY (device_id, ts_ms) ) '''); } @@ -3245,9 +3468,40 @@ class LocalDb { // on rec_ts is safe. `counter` is demoted to a NOT NULL forensic column (also // the keyset-cursor tiebreak in decodedOneHzBatchByRecTsRange, which never // fires now that rec_ts is unique). + // + // v47: KEYED BY (device_id, ts_ms), NOT rec_ts alone. `rec_ts` stays as the + // indexed READ key — every query in this file and in health_export ranges + // over it — but it can no longer be the identity, because a second device + // measuring the same second is a DIFFERENT reading, and REPLACE on a + // shared rec_ts silently deletes the first one (raw prunes at 3 days, so + // that loss is permanent). See [_rekeyTableByDevice]. + // + // `device_id = ''` IS RESERVED PERMANENTLY FOR THE PRIMARY BAND. Not a + // migration default — a standing rule, and it is load-bearing twice over: + // * The newest-wins dedupe above only works if every row from the one + // physical band shares one key value. A post-reboot counter reset and a + // re-drained record must still collide on (device_id, ts_ms). + // * A BLE `remoteId` is NOT STABLE (per-app CBPeripheral UUID on iOS, a + // rotating RPA on Android). Letting one reach this column would + // fragment one band into N identities across reinstalls, each with its + // own baseline. Only a SECONDARY device gets a real id, and only one an + // adapter issues from something the band emits across the handshake. + // + // `ts_ms` is `rec_ts * 1000` for every WHOOP row and for every row this + // migration rewrites, so the key is EXACTLY as unique as it was — no + // number moves. The millisecond resolution is there so a source faster + // than 1 Hz has somewhere to land instead of REPLACE-ing itself down to + // one row per second, which is the same defect in a different table. await db.execute(''' CREATE TABLE IF NOT EXISTS decoded_onehz ( - rec_ts INTEGER PRIMARY KEY, + device_id TEXT NOT NULL DEFAULT '', + ts_ms INTEGER NOT NULL DEFAULT 0, + -- NOT re-asserted NOT NULL, deliberately: [_rekeyTableByDevice] copies + -- this column verbatim out of a table where SQLite never enforced it + -- (a declared PRIMARY KEY on a legacy rowid table does not imply NOT + -- NULL), and a constraint failure inside onUpgrade's one exclusive + -- transaction quarantines the whole database (invariant 11). + rec_ts INTEGER, counter INTEGER NOT NULL, hr INTEGER, ax REAL, @@ -3269,7 +3523,8 @@ class LocalDb { temp_ch2_c REAL, temp_ch3_c REAL, signal_quality_logvar REAL, - dyn_accel_g REAL + dyn_accel_g REAL, + PRIMARY KEY (device_id, ts_ms) ) '''); // v43: `hr` IS NULLABLE TOO, and it is the last sensor column to get there. @@ -3320,23 +3575,58 @@ class LocalDb { // and the inserts were NON-SEQUENTIAL because `counter` resets to ~0 on a // band reboot — so it was paying page splits on the ingest path for a // query nobody makes. Dropped for existing installs in _repairOpenSchema. - // decoded_rr shares the rec_ts key with its parent: PRIMARY KEY (rec_ts, - // beat_index). Parent and child now delete/replace by the SAME key, so no - // orphan guard is needed. rr_ts_ms (= rec_ts*1000) stays as the per-beat - // timestamp the compute worker reads. No secondary index: the rec_ts-range - // read path is served by the PK, and the old UNIQUE(rr_ts_ms, beat_index) is - // now implied by the PK (rr_ts_ms is rec_ts*1000). + // decoded_rr shares its parent's key EXACTLY, one level deeper: PRIMARY KEY + // (device_id, ts_ms, beat_index) against the parent's (device_id, ts_ms). + // Parent and child still delete/replace by the SAME key, so no orphan guard + // is needed — and that is the whole reason the two keys must move together. + // Before v47 `_queueRrBeats` cleared the second with an unscoped + // `DELETE ... WHERE rec_ts = ?`, which for a second device deleted the + // FIRST band's beats for that second and could not be undone. + // rr_ts_ms (= rec_ts*1000) stays as the per-beat timestamp the compute + // worker reads; `rec_ts` stays as the indexed range key. await db.execute(''' CREATE TABLE IF NOT EXISTS decoded_rr ( + device_id TEXT NOT NULL DEFAULT '', + ts_ms INTEGER NOT NULL DEFAULT 0, rec_ts INTEGER NOT NULL, beat_index INTEGER NOT NULL, rr_ts_ms INTEGER NOT NULL, rr_ms INTEGER NOT NULL, device_family TEXT, source TEXT, - PRIMARY KEY (rec_ts, beat_index) + PRIMARY KEY (device_id, ts_ms, beat_index) ) '''); + // rec_ts USED TO BE THE KEY, and the key was the only index — every read in + // this file and in health_export ranges over it. Without these the v47 + // re-key turns each of them into a full table scan. + // + // GUARDED ON THE COLUMN, not on IF NOT EXISTS. This helper runs MID-LADDER + // too (`_rekeyDecodedStoreByRecTs`, oldV<33), where `decoded_rr` is still + // the counter-keyed shape and has no `rec_ts` at all — and an index on a + // missing column throws inside onUpgrade's one exclusive transaction and + // quarantines the whole database (invariant 11). + // + // TWO COLUMNS EACH, matching each table's `ORDER BY` exactly — `rec_ts, + // counter` on the parent and `rec_ts, beat_index` on the child — so the + // ordering comes out of the index and the planner needs no temp b-tree. + // + // ponytail: rec_ts kept as a second, indexed time axis. `ts_ms` is + // `rec_ts * 1000` for every row this app writes, so these reads COULD range + // on ts_ms and ride the PK for free — measured ~4.5 MB per 3-day store per + // index, on the hottest write path. Not done here because ~20 call sites in + // this file plus health_export range on rec_ts, and phase 2 is meant to + // change no read. Fold it in when a reader is being touched anyway. + for (final t in const { + 'decoded_onehz': 'rec_ts, counter', + 'decoded_rr': 'rec_ts, beat_index', + }.entries) { + if ((await _columnsOf(db, t.key)).contains('rec_ts')) { + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_${t.key}_rects ON ${t.key}(${t.value})', + ); + } + } // Existing installs get these here (ADD COLUMN, idempotent). await _ensureDeviceFamilyColumns(db); await _ensureSourceColumns(db); @@ -3524,6 +3814,138 @@ class LocalDb { await db.execute('ALTER TABLE _decoded_onehz_v39 RENAME TO decoded_onehz'); } + /// One `CREATE TABLE (…)` body reconstructed from a live `PRAGMA + /// table_info`, for the rebuild-aside migrations in this file. + /// + /// DERIVED, NEVER HARDCODED. These tables' column sets are genuinely in flux + /// (`ambient_raw` v42, `device_family` v41, `source` + the MT-12 channels + /// v43, `ts_subsec` / `band_sleep_state` off-ladder), and a hardcoded list + /// silently DROPS every column added after it was written — data loss, in a + /// rung that runs once and cannot be re-run. + /// + /// THE PRIMARY KEY IS THE PART THAT NEEDS CARE. `PRAGMA table_info.pk` is a + /// 1-based POSITION, not a flag, so a composite key reports several columns. + /// A one-column key is emitted INLINE (`x INTEGER PRIMARY KEY`) because that + /// is the only spelling that keeps SQLite's rowid-alias behaviour; anything + /// wider must be a table-level `PRIMARY KEY (a, b)` clause, and emitting the + /// inline form for each of them instead throws "table has more than one + /// primary key". + /// + /// [prepend] columns are emitted first, verbatim, and are NOT in [info] — a + /// re-key adds its new key columns that way. [primaryKey] replaces the old + /// key entirely; omit it to carry the table's own key across. + /// [dropNotNull] relaxes named columns. Nothing here ever ADDS a constraint: + /// a constraint failure inside `onUpgrade`'s single exclusive transaction + /// rolls the whole ladder back and quarantines the database (invariant 11), + /// so a rebuild may only ever loosen. + static String _rebuildDdlBody( + List> info, { + Set dropNotNull = const {}, + List prepend = const [], + List? primaryKey, + }) { + final own = [ + for (final c in info) + if ((((c['pk'] as num?)?.toInt()) ?? 0) > 0) c, + ]..sort( + (a, b) => ((a['pk'] as num).toInt()).compareTo((b['pk'] as num).toInt()), + ); + final key = primaryKey ?? [for (final c in own) c['name'] as String]; + final inline = key.length == 1 ? key.first : null; + final defs = [...prepend]; + for (final c in info) { + final name = c['name'] as String; + final dflt = c['dflt_value']; + final notNull = + (((c['notnull'] as num?)?.toInt()) ?? 0) == 1 && + !dropNotNull.contains(name); + defs.add( + '$name ${(c['type'] as String?) ?? ''}' + '${name == inline ? ' PRIMARY KEY' : ''}' + '${notNull ? ' NOT NULL' : ''}' + '${dflt == null ? '' : ' DEFAULT $dflt'}', + ); + } + if (inline == null && key.isNotEmpty) { + defs.add('PRIMARY KEY (${key.join(', ')})'); + } + return defs.join(', '); + } + + /// v47: put `device_id` in front of the key of every table whose identity was + /// a WHOOP-shaped quantity — a record second, or the band's flash counter. + /// + /// WHY THIS ONE IS NOT DEFERRABLE. `decoded_onehz` was `rec_ts INTEGER + /// PRIMARY KEY` written with REPLACE and `decoded_rr` was cleared by an + /// unscoped `DELETE … WHERE rec_ts = ?`, so a second device measuring the + /// same second did not merge with the first — it DELETED it, row and beats. + /// `raw_archive` prunes at `rawRetentionDays = 3`, so within three days the + /// bytes that could rebuild the evicted row are gone too. Every other item on + /// the band-agnostic roadmap can be done after a second device has written; + /// this one cannot. + /// + /// WHAT IT DOES NOT CHANGE. Every existing row is copied under + /// `device_id = ''` with `ts_ms = rec_ts * 1000`, which is exactly as unique + /// as `rec_ts` was — one row per second per device, same newest-wins REPLACE, + /// same dedupe. No stored value is rewritten and no derived number moves, so + /// there is no `kAlgoVersion` bump with it. + /// + /// Cheap enough for the launch-path CPU watchdog `onUpgrade` runs inside: + /// the decoded store is retention-capped at `rawRetentionDays`, ~260 k rows, + /// and every copy is a server-side `INSERT … SELECT` with zero host-bound + /// variables (so the iOS `SQLITE_MAX_VARIABLE_NUMBER` never applies and no + /// chunking is needed). + static Future _rekeyStoresByDeviceId(Database db) async { + await _rekeyTableByDevice(db, 'decoded_onehz'); + await _rekeyTableByDevice(db, 'decoded_rr', keyTail: const ['beat_index']); + await _rekeyTableByDevice(db, 'samples', timeCol: 'ts'); + } + + /// The rename-aside rebuild behind [_rekeyStoresByDeviceId], for one table. + /// + /// Self-skipping and idempotent: a table that already carries `device_id` is + /// left alone (so a fresh install at v47+ does no work at all), the temp + /// table is dropped up front so a crash mid-migration re-runs cleanly, and + /// the copy is keyed on identity. + static Future _rekeyTableByDevice( + Database db, + String table, { + String timeCol = 'rec_ts', + List keyTail = const [], + }) async { + final info = await db.rawQuery('PRAGMA table_info($table)'); + // Table absent on this upgrade path ⇒ the current DDL already carries the + // new key and there is nothing to rebuild. + if (info.isEmpty) return; + final names = [for (final c in info) c['name'] as String]; + if (names.contains('device_id')) return; + final tmp = '_${table}_v47'; + await db.execute('DROP TABLE IF EXISTS $tmp'); + await db.execute( + 'CREATE TABLE $tmp (${_rebuildDdlBody( + info, + prepend: const [ + "device_id TEXT NOT NULL DEFAULT ''", + 'ts_ms INTEGER NOT NULL DEFAULT 0', + ], + primaryKey: ['device_id', 'ts_ms', ...keyTail], + )})', + ); + final cols = names.join(', '); + // COALESCE because a declared PRIMARY KEY on a legacy rowid table does NOT + // enforce NOT NULL — a NULL time column would put a NULL in the new key, + // where it compares unequal to itself and escapes the retention prune + // forever. 0 is behind every cutoff, so such a row is pruned on the next + // pass instead of leaking. + await db.execute( + 'INSERT OR REPLACE INTO $tmp (device_id, ts_ms, $cols) ' + "SELECT '', COALESCE($timeCol, 0) * 1000, $cols " + 'FROM $table ORDER BY $timeCol ASC', + ); + await db.execute('DROP TABLE IF EXISTS $table'); + await db.execute('ALTER TABLE $tmp RENAME TO $table'); + } + /// v43 (SLP-05): drop `NOT NULL` from `decoded_onehz.hr`. /// /// See [_createDecodedStore] for WHY. This is the same rebuild shape as @@ -3549,24 +3971,11 @@ class LocalDb { final hr = info.where((c) => c['name'] == 'hr'); if (hr.isEmpty || (hr.first['notnull'] as num?)?.toInt() != 1) return; - String defOf(Map c) { - final name = c['name'] as String; - final type = (c['type'] as String?) ?? ''; - final dflt = c['dflt_value']; - // decoded_onehz's PK is the single column rec_ts, so the inline form is - // exact. `hr` is the one column that loses its NOT NULL. - final pk = ((c['pk'] as num?)?.toInt() ?? 0) > 0; - final notNull = - ((c['notnull'] as num?)?.toInt() ?? 0) == 1 && name != 'hr'; - return '$name $type${pk ? ' PRIMARY KEY' : ''}' - '${notNull ? ' NOT NULL' : ''}' - '${dflt == null ? '' : ' DEFAULT $dflt'}'; - } - final names = [for (final c in info) c['name'] as String]; await db.execute('DROP TABLE IF EXISTS _decoded_onehz_v43'); await db.execute( - 'CREATE TABLE _decoded_onehz_v43 (${info.map(defOf).join(', ')})', + 'CREATE TABLE _decoded_onehz_v43 ' + '(${_rebuildDdlBody(info, dropNotNull: const {'hr'})})', ); final cols = names.join(', '); await db.execute( @@ -3913,6 +4322,20 @@ class LocalDb { RawRecord raw, Sample? sample, { String? deviceFamily, + String deviceId = kPrimaryDeviceId, + // TRUE ONLY FROM A MID-LADDER REPLAY, and it is not a style choice. This + // map is also written by `_backfillDecodedStore` (rungs oldV<11 / oldV<20) + // and `redriveArchivedRecords` (rung oldV<44), both of which run BEFORE the + // v47 rung hands `device_id` / `ts_ms` back — and naming a column that does + // not exist yet throws inside onUpgrade's one exclusive transaction and + // quarantines the whole database. Their rows are re-keyed by + // [_rekeyTableByDevice] a few rungs later, from `rec_ts`, which is the same + // value this path would have written. + // + // The default is FALSE and the key columns are named, deliberately: the + // opposite default would let a forgotten argument write every row at + // ('', 0), where REPLACE collapses the entire store to ONE row. + bool preDeviceKey = false, }) { final decoded = _decodeOneHzSample(raw, preferred: sample); if (decoded == null) { @@ -3929,6 +4352,8 @@ class LocalDb { _recTsFrom(raw, sample), sample, deviceFamily: deviceFamily, + deviceId: deviceId, + preDeviceKey: preDeviceKey, ); } return 0; @@ -3943,6 +4368,16 @@ class LocalDb { // can no longer make one second's record evict another's (the pre-fix // counter-PK eviction that silently, unrecoverably deleted 1 Hz rows). batch.insert('decoded_onehz', { + // v47: WHICH DEVICE, in front of the key. '' is the primary band and + // nothing else may ever use it — see _createDecodedStore for why an + // unstable BLE remoteId must never reach this column. + 'device_id': ?(preDeviceKey ? null : deviceId), + // `rec_ts * 1000` EXACTLY, never `+ tsSubsec`. The key has to stay as + // unique as `rec_ts` was or the newest-wins dedupe splits into one row + // per sub-second and every count in the app changes. The millisecond + // resolution is headroom for a faster-than-1-Hz source, not a place to + // put this record's own sub-second (which already has `ts_subsec`). + 'ts_ms': ?(preDeviceKey ? null : recTs * 1000), 'rec_ts': recTs, 'counter': raw.counter, // v43: ABSENCE IS NULL HERE TOO. `hr == 0` is the off-skin sentinel, so @@ -4009,7 +4444,14 @@ class LocalDb { 'device_family': ?deviceFamily, }, conflictAlgorithm: ConflictAlgorithm.replace); return 1 /* the decoded_onehz insert */ + - _queueRrBeats(batch, recTs, decoded, deviceFamily: deviceFamily); + _queueRrBeats( + batch, + recTs, + decoded, + deviceFamily: deviceFamily, + deviceId: deviceId, + preDeviceKey: preDeviceKey, + ); } /// `rec_ts` for one raw+decoded pair. @@ -4026,61 +4468,6 @@ class LocalDb { return (rawRecTs != null && rawRecTs > 0) ? rawRecTs : decoded.tsEpoch; } - /// Where each beat in one record actually sits, in absolute epoch ms — or - /// null for a beat that cannot be placed. One entry per entry in [rrMs]. - /// - /// TWO PARTS, AND THEY ARE NOT EQUALLY SOLID. Read them separately. - /// - /// THE ANCHOR IS MEASURED. `rec_ts + tsSubsec/32768` is the record's own - /// timestamp, whole seconds and sub-second, exactly as the strap sent it. - /// This app has dropped the second half of that since forever, pinning every - /// record to a whole second. With no sub-second there is no anchor and every - /// beat here is null; a whole second is NOT substituted for one, because the - /// whole point of this column is to say something the old one could not. - /// - /// THE PLACEMENT IS A MODEL, and it is one assumption wide: an R-R interval - /// is the gap ENDING at its beat (that part is the definition), and the LAST - /// beat a record reports sits at the record's timestamp. Everything else - /// follows — beat i is the anchor minus the intervals after it. The direction - /// is chosen because backwards is the only one that cannot place a beat in - /// the future, i.e. after the moment we were told about it; a forward walk - /// would also run every multi-beat record past its own second (the intervals - /// sum to 1,426 ms on a 2-beat record and 2,594 ms on a 4-beat one, measured) - /// and straight through the next record's. - /// - /// WHAT THE INTERVALS DO NOT DO IS TILE THE SECOND. Over 81 uninterrupted - /// runs of 300+ consecutive records in a real export, the intervals sum to - /// 0.967 of the `rec_ts` span (0.960-0.990 across runs) — so the beat train is - /// a CHAIN that runs a few percent short, which is what a handful of rejected - /// beats looks like, and not a set of per-second buckets. Several of a - /// record's intervals reach back out of its own second. Per-record placement - /// is nevertheless what THIS path can do — records arrive batched, out of - /// order and with gaps, so no cross-record chain is available at the write — - /// and a consumer that wants the chain can walk `rr_ms` itself. - /// - /// WHAT MOVES AND WHAT DOES NOT. Nothing shipped changes: time-domain HRV - /// (RMSSD, SDNN, pNNx) is built out of interval VALUES, which were always - /// right, and no reader of this column exists. What it makes possible is - /// anything needing absolute placement — a Lomb-Scargle periodogram handed - /// beats where they happened instead of a staircase, a beat put on the same - /// axis as a motion sample, a real inter-record gap. - /// - /// A non-positive interval BREAKS THE CHAIN: the gap before that beat is - /// unknown, so every EARLIER beat in the record becomes unplaceable and gets - /// null rather than a position computed as if the missing gap were zero. - @visibleForTesting - static List beatTimesMs(int recTs, int? tsSubsec, List rrMs) { - final out = List.filled(rrMs.length, null); - if (tsSubsec == null || rrMs.isEmpty) return out; - final anchor = recTs * 1000 + (tsSubsec * 1000) ~/ 32768; - var back = 0; - for (var i = rrMs.length - 1; i >= 0; i--) { - out[i] = anchor - back; - if (rrMs[i] <= 0) break; - back += rrMs[i]; - } - return out; - } /// Replaces this second's RR beats. Returns the ops queued. /// @@ -4092,14 +4479,31 @@ class LocalDb { int recTs, Sample decoded, { String? deviceFamily, + String deviceId = kPrimaryDeviceId, + bool preDeviceKey = false, }) { - batch.rawDelete('DELETE FROM decoded_rr WHERE rec_ts = ?', [recTs]); + // SCOPED TO THE WRITING DEVICE (v47). Unscoped, this cleared every device's + // beats for the second — so a second band writing one row deleted the + // first band's R-R for that second, permanently (raw prunes at 3 days). + // Same key prefix as the parent row, so the PK serves the delete. + if (preDeviceKey) { + batch.rawDelete('DELETE FROM decoded_rr WHERE rec_ts = ?', [recTs]); + } else { + batch.rawDelete( + 'DELETE FROM decoded_rr WHERE device_id = ? AND ts_ms = ?', + [deviceId, recTs * 1000], + ); + } final beatTs = beatTimesMs(recTs, decoded.tsSubsec, decoded.rrIntervalsMs); var ops = 1; for (var i = 0; i < decoded.rrIntervalsMs.length; i++) { final rr = decoded.rrIntervalsMs[i]; if (rr <= 0) continue; batch.insert('decoded_rr', { + // Same key prefix as the parent row — see _createDecodedStore. Omitted + // on the mid-ladder replay for the reason _queueDecodedOneHz gives. + 'device_id': ?(preDeviceKey ? null : deviceId), + 'ts_ms': ?(preDeviceKey ? null : recTs * 1000), 'rec_ts': recTs, 'beat_index': i, // UNCHANGED, DELIBERATELY. `rr_ts_ms` stays `rec_ts * 1000` and @@ -4112,9 +4516,10 @@ class LocalDb { // NULL is "we did not keep the sub-second for this beat", which is // exactly the case for every row written before today. // - // It is also the column the compute layer reads (substrate.dart, - // derive_prepare.dart), and this change is deliberately invisible to - // it — see beatTimesMs on why no shipped number moves. + // The compute layer (substrate.dart, derive_prepare.dart) now prefers + // `beat_ts_ms` and falls back to this column, so `rr_ts_ms` stays the + // honest answer for every row that has no sub-second — which is every + // row written before the column existed. 'rr_ts_ms': recTs * 1000, 'rr_ms': rr, // Omitted when null — see _queueDecodedOneHz. Also newer than the @@ -4167,7 +4572,9 @@ class LocalDb { capturedAt: (row['captured_at'] as num?)?.toInt() ?? 0, recTs: (row['rec_ts'] as num?)?.toInt(), ); - _queueDecodedOneHz(batch, raw, null); + // MID-LADDER: `device_id` / `ts_ms` do not exist yet (v47 rung). See + // _queueDecodedOneHz's `preDeviceKey`. + _queueDecodedOneHz(batch, raw, null, preDeviceKey: true); } await batch.commit(noResult: true); afterCounter = (rows.last['counter'] as num?)?.toInt() ?? afterCounter; @@ -4729,6 +5136,12 @@ class LocalDb { ); if (present.isEmpty) return 0; } + // SELF-DETECTING, because this runs from BOTH sides of the v47 rung: from + // the oldV<44 ladder step, where `decoded_onehz` is still keyed by rec_ts + // alone and naming `device_id` would throw inside onUpgrade (quarantining + // the database), and from the app/tests on a re-keyed table. One PRAGMA. + final preDeviceKey = + !(await _columnsOf(db, 'decoded_onehz')).contains('device_id'); final marks = List.filled(redrivableArchiveReasons.length, '?').join(','); // Paged on `hex`, which is the table's PRIMARY KEY — a stable, total order @@ -4793,7 +5206,13 @@ class LocalDb { // `sample` is handed back as `preferred` so the hex is decoded once, // not twice; `_queueDecodedOneHz` returns it straight back out. // `deviceFamily` is deliberately omitted — see the doc comment. - if (_queueDecodedOneHz(batch, raw, sample) > 0) { + if (_queueDecodedOneHz( + batch, + raw, + sample, + preDeviceKey: preDeviceKey, + ) > + 0) { queued++; recovered++; } @@ -5293,7 +5712,12 @@ class LocalDb { final lo = fromRecTs <= toRecTs ? fromRecTs : toRecTs; final hi = fromRecTs <= toRecTs ? toRecTs : fromRecTs; return db.rawQuery( - 'SELECT rec_ts, beat_index, rr_ts_ms, rr_ms FROM decoded_rr ' + // `beat_ts_ms` is the MEASURED beat instant (`beatTimesMs`); `rr_ts_ms` is + // `rec_ts * 1000`, a whole-second staircase that says every beat inside + // one record happened at the same millisecond. Both are returned because + // `beat_ts_ms` is NULL on every row banked before the column existed and + // on every source that carries no sub-second — the reader coalesces. + 'SELECT rec_ts, beat_index, rr_ts_ms, rr_ms, beat_ts_ms FROM decoded_rr ' 'WHERE rec_ts >= ? AND rec_ts <= ? AND source IS NULL ' 'ORDER BY rec_ts ASC, beat_index ASC', [lo, hi], @@ -5331,6 +5755,11 @@ class LocalDb { // the column is that a guessed provenance is worse than none. See // [_createMetricSeriesVersion]. String? source, + // WHICH BAND'S UNITS these scalars are in — the substrate's own + // `device_family` stamp, which is already null when the window spans two + // straps or carries no stamp at all. Same rule as [source]: never guessed. + // See [_createMetricSeriesVersion] and [foreignFamilyDates]. + String? deviceFamily, }) async { final db = await instance; final now = DateTime.now().millisecondsSinceEpoch; @@ -5385,6 +5814,7 @@ class LocalDb { // included, so a caller that does not know its own writes NULL // here rather than inheriting the previous writer's claim. 'source': source, + 'device_family': deviceFamily, }, conflictAlgorithm: ConflictAlgorithm.replace); } } @@ -6207,6 +6637,45 @@ class LocalDb { }; } + /// The `metric_series` keys whose stored VALUES ARE IN THE BAND'S OWN UNITS, + /// so a day measured by a different band family is not on the same scale as + /// today's and must not sit in the same baseline. + /// + /// DELIBERATELY SHORT, and it is a list of UNIT MISMATCHES, not of things + /// that got noisier. `skin_temp_adc` is a gen4 thermistor ADC COUNT on one + /// side of a strap swap and a gen5 CENTI-DEGREE reading on the other — the + /// same key holding two different quantities, which is a wrong number, not a + /// less precise one. `rhr` and `rmssd` off a different wrist sensor are the + /// same quantity measured slightly differently: masking those would delete + /// the user's history from their own baseline to fix a bias smaller than the + /// window it is measured over. Nothing joins this set without a measured unit + /// difference behind it. + static const Set familySeamKeys = {'skin_temp_adc'}; + + /// Day labels whose scalars were measured by a DIFFERENT band family than the + /// most recent stamped day — the mask for [familySeamKeys] baselines. + /// + /// Two properties that make this safe to apply unconditionally: + /// * A NULL stamp is never foreign. Unknown provenance is its own case (see + /// [_createMetricSeriesVersion]); every day written before the column + /// existed reads NULL, and dropping those would delete a real user's whole + /// history from their own baseline. + /// * A single-family user gets the EMPTY SET, because nothing differs from + /// the newest stamp. So this changes no number until a strap actually + /// changes generation. + static Future> foreignFamilyDates() async { + final db = await instance; + final rows = await db.rawQuery( + 'SELECT date FROM metric_series_version ' + 'WHERE date IS NOT NULL AND device_family IS NOT NULL ' + 'AND device_family <> (' + ' SELECT device_family FROM metric_series_version ' + ' WHERE device_family IS NOT NULL ORDER BY date DESC LIMIT 1' + ')', + ); + return {for (final r in rows) r['date'] as String}; + } + /// Import another device's exported OpenStrap DB ([path], from [exportCopy] + /// share) by MERGING its rows into this one (INSERT-OR-REPLACE). Covers derived /// results, the metric series, user data, and the raw ledger so the receiving @@ -6287,6 +6756,11 @@ class LocalDb { 'cycle_log', 'cycle_symptom', 'breathing_session', + // Vendor-computed, typed-in and imported scalars. In the hand-entered + // block because a third of it IS hand-entered and nothing regenerates + // any of it — a `reports` band trims its own history, and the app whose + // export the imported rows came from may be uninstalled by now. + 'observation', 'workout_route', 'workout_split', // The user's sleep corrections. These are the ONLY copy of them — the @@ -6490,6 +6964,22 @@ class LocalDb { } row['rec_ts'] = rrTsMs.toInt() ~/ 1000; } + // A PRE-v47 EXPORT CARRIES NO `device_id` / `ts_ms`. Left + // alone they take the column DEFAULTS — ('', 0) — so every + // imported row would collide on ONE primary key and REPLACE + // the whole table down to a single row. Derive the key exactly + // as [_rekeyTableByDevice] does: the primary band, and the + // row's own time in ms. AFTER the rec_ts recovery above, which + // is what decoded_rr's key is built from. + if (cols.contains('ts_ms') && row['ts_ms'] == null) { + row['device_id'] ??= kPrimaryDeviceId; + final t0 = row[t == 'samples' ? 'ts' : 'rec_ts']; + // Non-numeric (storage classes are per VALUE in a foreign + // export) ⇒ drop the one row rather than key it at 0, where + // it would REPLACE another row that genuinely belongs there. + if (t0 is! num) continue; + row['ts_ms'] = t0.toInt() * 1000; + } rows.add(row); } // REPLACE the beat set for a colliding second, don't patch it. @@ -6509,7 +6999,10 @@ class LocalDb { // clears >1; page 2 inserts 2,3 and clears >3 — and beat_index is // dense by construction, so "everything past the last one" is // exactly the stale local tail. - final highestBeat = {}; + // v47: keyed on (device_id, ts_ms), the beat's real key — so the + // tail sweep can only ever clear the WRITING device's stale + // beats, never the other band's for the same second. + final highestBeat = <(Object?, Object?), int>{}; for (final row in rows) { batch.insert( t, @@ -6518,12 +7011,12 @@ class LocalDb { ); copied++; if (t == 'decoded_rr') { - final recTs = row['rec_ts']; final idx = row['beat_index']; - if (recTs != null && idx is num) { + final key = (row['device_id'], row['ts_ms']); + if (key.$2 != null && idx is num) { final n = idx.toInt(); - final prev = highestBeat[recTs]; - if (prev == null || n > prev) highestBeat[recTs] = n; + final prev = highestBeat[key]; + if (prev == null || n > prev) highestBeat[key] = n; } } if (++ops >= chunkOps) await flush(); @@ -6531,8 +7024,8 @@ class LocalDb { for (final e in highestBeat.entries) { batch.delete( 'decoded_rr', - where: 'rec_ts = ? AND beat_index > ?', - whereArgs: [e.key, e.value], + where: 'device_id = ? AND ts_ms = ? AND beat_index > ?', + whereArgs: [e.key.$1, e.key.$2, e.value], ); if (++ops >= chunkOps) await flush(); } @@ -6753,6 +7246,7 @@ class LocalDb { 'external_hr', 'imported_measurement', 'imported_workout', + 'observation', ]; final missingTables = []; @@ -8315,6 +8809,13 @@ class LocalDb { /// 1 Hz heart-rate samples in [fromTs, toTs] (epoch SECONDS), ascending, used /// to colour a route and average HR per split. Only worn seconds (hr > 0). + /// + /// Band rows only (`source IS NULL`), for the same reason [sessionHrStats] + /// states below: a paired chest strap writes its own seconds into this table + /// with `source` set, and averaging the two together reports a session HR that + /// is neither sensor's. Today nothing writes a non-null source, so this filter + /// is a no-op — it is here so the six call sites are already correct on the + /// day one does. static Future>> hrSamplesInRange( int fromTs, int toTs, @@ -8323,7 +8824,7 @@ class LocalDb { return db.query( 'decoded_onehz', columns: ['rec_ts', 'hr'], - where: 'rec_ts >= ? AND rec_ts <= ? AND hr > 0', + where: 'rec_ts >= ? AND rec_ts <= ? AND hr > 0 AND source IS NULL', whereArgs: [fromTs, toTs], orderBy: 'rec_ts ASC', ); diff --git a/lib/data/live_coverage_policy.dart b/lib/data/live_coverage_policy.dart index 2da4c907..e6be5551 100644 --- a/lib/data/live_coverage_policy.dart +++ b/lib/data/live_coverage_policy.dart @@ -186,6 +186,7 @@ class CoverageSpan { required this.endTs, required this.steps, required this.fromBand, + this.deviceId = '', }); final int startTs; @@ -194,6 +195,15 @@ class CoverageSpan { /// `source != 'phone'` — the band's own 100 Hz pedometer, live or imported. final bool fromBand; + + /// WHICH PHYSICAL SENSOR counted these steps. `''` is permanently reserved + /// for the primary band (and for the phone, which is likewise one of it), + /// so a single-device install is entirely `''` and behaves exactly as it did + /// before this field existed. + /// + /// It exists because two sensors that are not the same sensor can produce + /// two spans of the same rank over the same walk — see [resolveDaySteps]. + final String deviceId; } /// A day's steps after the ladder has run, split by the sensor that counted. @@ -235,10 +245,12 @@ class ResolvedDaySteps { } class _Ranked { - _Ranked(this.startTs, this.endTs, this.steps, this.rank, this.fromBand); + _Ranked(this.startTs, this.endTs, this.steps, this.rank, this.fromBand, + this.deviceId); final int startTs, endTs, rank; final double steps; final bool fromBand; + final String deviceId; double credited = 0; } @@ -263,10 +275,19 @@ ResolvedDaySteps resolveDaySteps(Iterable rows) { // 2 = band that looks like gait, 1 = phone, 0 = band that does not. final rank = r.fromBand ? (spm >= kBandSpanMinSpm ? 2 : 0) : 1; spans.add( - _Ranked(w.startTs, w.endTs, r.steps.toDouble(), rank, r.fromBand), + _Ranked(w.startTs, w.endTs, r.steps.toDouble(), rank, r.fromBand, + r.deviceId), ); } - spans.sort((a, b) => b.rank.compareTo(a.rank)); + // Rank first, then start time — `List.sort` is NOT stable in Dart, and once + // equal-ranked spans from different devices compete (below) the winner is + // whichever is resolved first, so the tie needs a rule of its own or the + // day's total depends on the order SQLite happened to return the rows in. + // Earliest span wins; it is arbitrary but it is the same arbitrary answer + // every run, which is what a re-derive needs. + spans.sort((a, b) => b.rank != a.rank + ? b.rank.compareTo(a.rank) + : a.startTs.compareTo(b.startTs)); var strap = 0.0; var phone = 0.0; @@ -275,10 +296,17 @@ ResolvedDaySteps resolveDaySteps(Iterable rows) { var alreadyCounted = 0.0; for (var j = 0; j < i; j++) { final h = spans[j]; - // EQUAL RANK IS NOT COMPETITION. Two band rows, or two phone rows, are - // the same sensor reporting twice and are summed — which is what the - // table has always meant and what re-import idempotency relies on. - if (h.rank <= s.rank) continue; + // EQUAL RANK IS NOT COMPETITION *WITHIN ONE DEVICE*. Two band rows, or + // two phone rows, from the SAME `device_id` are the same sensor reporting + // twice and are summed — which is what the table has always meant and + // what re-import idempotency relies on. + // + // Two DIFFERENT devices are not that. Rank is a property of the SPAN + // (band-that-looks-like-gait / phone / band-that-does-not), so two straps + // on the same 3,000-step walk both rank 2, skipped each other here, and + // the day published 6,000. Different id ⇒ they compete like any other + // pair, and the tie is broken by start time (see the sort above). + if (h.rank <= s.rank && h.deviceId == s.deviceId) continue; final ov = math.min(s.endTs, h.endTs) - math.max(s.startTs, h.startTs); if (ov <= 0) continue; alreadyCounted += h.credited * ov / (h.endTs - h.startTs); @@ -301,6 +329,7 @@ ResolvedDaySteps resolveDaySteps(Iterable rows) { endTs: s.endTs, steps: s.credited.round(), fromBand: s.fromBand, + deviceId: s.deviceId, ), ]..sort((a, b) => a.startTs.compareTo(b.startTs)), ); diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index 212f0604..eac6cdbb 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -3771,7 +3771,8 @@ class LocalRepositoryImpl extends LocalRepository { // enough heart rate" — and offered "wear the band for your normal hard // sessions". Measured on all three real databases, both are false: every // row is unstamped (`unknown_device_family:id=none` on `hr_ceiling`). - final ceilingNote = ceiling == null && ana.deviceFamilyOf(family) == null + final ceilingNote = ceiling == null && + ana.calibrationFor(ana.hrCeilingMotionGateG, family) == null ? ana.unknownFamilyNote(family) : null; final set = trainingZones( diff --git a/lib/data/models.dart b/lib/data/models.dart index cce9a7c6..0d67a791 100644 --- a/lib/data/models.dart +++ b/lib/data/models.dart @@ -208,7 +208,6 @@ class Sample { bandSleepState: bandSleepState, ); - bool get wristOn => hr > 0; bool get hasDecodedOneHz => ax != null && ay != null && diff --git a/lib/data/observation.dart b/lib/data/observation.dart new file mode 100644 index 00000000..25f92749 --- /dev/null +++ b/lib/data/observation.dart @@ -0,0 +1,95 @@ +// observation.dart — the type for a number OpenStrap did not compute. +// +// Deliberately NOT `Metric`. `Metric` carries `tier`, `confidence` and +// `inputs_used`, and every one of those is a statement about OUR method: which +// published algorithm ran, over which inputs, with how much of the window +// present. A vendor's Body Battery has no method we can describe, a typed-in +// mood has none at all, and an Apple Health step count has someone else's. +// Giving any of them a tier would be inventing a provenance claim. +// +// Lives in `edge/` on purpose. `analytics/` never sees this type — it stays +// device-blind and computes only from raw signal (OBSERVATION_SPEC §4). +// +// NAME COLLISION, on purpose and worth knowing about: `ui2/grammar.dart` also +// declares an `Observation` — the clinician-worthy-pattern CARD. No file +// imports both today. A file that ever needs both must prefix one import; +// renaming either would be worse, because both names are the right word for +// what they are. + +import 'day_label.dart' show dayLabelOf; + +/// Where an observation came from — the `observation.source_kind` column. +/// +/// An enum rather than a bare string because this value is part of the row's +/// identity (it is in the unique index), so a typo does not fail, it forks the +/// row into a second one that never collides with the first. +/// +/// NOT derivable from the other fields, which is why it is carried explicitly: +/// an imported Apple Health step count and a typed-in mood both fill [key] and +/// leave [vendorKey] null, and they are not the same kind of thing. +enum ObservationSource { + /// The band computed it on-device and shipped the conclusion. + vendor, + + /// The user typed it in — food, a workout, mood, water. + entered, + + /// Another app's history, adopted wholesale (Apple Health, a Takeout). + imported, +} + +/// One scalar OpenStrap stores but did not derive. +/// +/// Exactly one of [key] and [vendorKey] identifies the row, and at least one +/// MUST be set — the storage layer's unique index is built on +/// `COALESCE(vendor_key, key)` and a row with neither has no identity at all. +/// +/// The split is the whole point, and it is the rule that stops `readiness` +/// meaning three different algorithms depending on which band was worn: +/// +/// * [key] — OUR vocabulary, and ONLY for a **comparable quantity**: the same +/// physical thing measured a different way (steps, sleep duration, resting +/// HR). +/// * [vendorKey] — THEIR name, verbatim, forever, for a **proprietary +/// composite** (Body Battery, PAI, Recovery %, a sleep score). Mapping one +/// of those onto one of our keys is the worst available mistake here. +class Observation { + const Observation({ + required this.sourceKind, + required this.value, + required this.attribution, + required this.at, + this.key, + this.vendorKey, + this.unit, + }) : assert( + key != null || vendorKey != null, + 'an observation with neither key nor vendorKey has no identity', + ); + + /// Our vocabulary — comparable quantities only. Null for a composite. + final String? key; + + /// Their name, verbatim. Null for anything that is not a vendor's own. + final String? vendorKey; + + final num value; + + /// Free text, as the vendor states it ('%', 'steps', 'bpm'). Null when the + /// number is unitless — a sleep score out of 100 is not a measurement of + /// anything and inventing a unit for it would imply it was. + final String? unit; + + /// What the user sees next to the number: 'Amazfit', 'you', 'Apple Health'. + /// A value without this is not renderable — the whole product posture for a + /// `reports` device is that their number is shown as theirs. + final String attribution; + + final DateTime at; + + /// The LOCAL calendar day this observation belongs to — the same day model + /// every other table is keyed by, via the one helper that gets DST right. + String get date => dayLabelOf(at); + + final ObservationSource sourceKind; +} diff --git a/test/absence_and_offload_guards_test.dart b/test/absence_and_offload_guards_test.dart index a5d24d6d..527e7a4c 100644 --- a/test/absence_and_offload_guards_test.dart +++ b/test/absence_and_offload_guards_test.dart @@ -70,25 +70,25 @@ void main() { }); }); - group('plausibleHrOrZero — the gen4 trusted path had no upper bound', () { + group('plausibleHrOrNull — the gen4 trusted path had no upper bound', () { test('a physiological HR passes through untouched', () { - expect(plausibleHrOrZero(60), 60); - expect(plausibleHrOrZero(kMinPlausibleHr), kMinPlausibleHr); - expect(plausibleHrOrZero(kMaxPlausibleHr), kMaxPlausibleHr); + expect(plausibleHrOrNull(60), 60); + expect(plausibleHrOrNull(kMinPlausibleHr), kMinPlausibleHr); + expect(plausibleHrOrNull(kMaxPlausibleHr), kMaxPlausibleHr); }); test('an impossible HR reads ABSENT, and is never clamped into range', () { // 250 used to pass the `hr > 0` filter and become the displayed max HR. - expect(plausibleHrOrZero(250), 0); - expect(plausibleHrOrZero(255), 0); - expect(plausibleHrOrZero(7), 0); + expect(plausibleHrOrNull(250), isNull); + expect(plausibleHrOrNull(255), isNull); + expect(plausibleHrOrNull(7), isNull); // Clamping would have produced a plausible-looking 230/25 — a number the // wrist never reported. - expect(plausibleHrOrZero(250), isNot(kMaxPlausibleHr)); + expect(plausibleHrOrNull(250), isNot(kMaxPlausibleHr)); }); - test('the off-skin sentinel is preserved', () { - expect(plausibleHrOrZero(0), 0); + test('a zero reading is not a heart rate either', () { + expect(plausibleHrOrNull(0), isNull); }); }); diff --git a/test/band_step_counter_test.dart b/test/band_step_counter_test.dart index 7a0aa6c3..3644fd9c 100644 --- a/test/band_step_counter_test.dart +++ b/test/band_step_counter_test.dart @@ -38,22 +38,22 @@ void main() { test('gen4 (no counter on any record) returns NULL, not zero', () { // The distinction the whole feature rests on: "this hardware cannot count // steps" must not render as "you took no steps". - expect(hardwareStepsFromCounter(_sub([-1, -1, -1, -1])), isNull); - expect(hardwareStepsFromCounter(Substrate.empty), isNull); + expect(hardwareStepsFromCounter(_sub([-1, -1, -1, -1]), cumulativeCounterModulus: 65536), isNull); + expect(hardwareStepsFromCounter(Substrate.empty, cumulativeCounterModulus: 65536), isNull); }); test('a counter that never moves is a real, confident ZERO', () { - expect(hardwareStepsFromCounter(_sub([4000, 4000, 4000])), 0); + expect(hardwareStepsFromCounter(_sub([4000, 4000, 4000]), cumulativeCounterModulus: 65536), 0); }); test('sums positive deltas, not last minus first', () { - expect(hardwareStepsFromCounter(_sub([100, 102, 105, 105, 109])), 9); + expect(hardwareStepsFromCounter(_sub([100, 102, 105, 105, 109]), cumulativeCounterModulus: 65536), 9); }); test('a u16 WRAP is recovered, not lost and never negative', () { // 65530 -> 3 is a 9-step delta through the wrap. Read naively it is // -65527, which would drive the total negative. - final steps = hardwareStepsFromCounter(_sub([65520, 65530, 3, 6])); + final steps = hardwareStepsFromCounter(_sub([65520, 65530, 3, 6]), cumulativeCounterModulus: 65536); expect(steps, 10 + 9 + 3); expect(steps, isNonNegative); }); @@ -62,32 +62,33 @@ void main() { // Reboot mid-day: 40000 -> 0. Modulo 65536 that reads as 25536 steps in // one second, which fails the plausibility budget and is dropped whole. // The steps after the reset still count. - expect(hardwareStepsFromCounter(_sub([39998, 40000, 0, 5, 9])), 2 + 5 + 4); + expect(hardwareStepsFromCounter(_sub([39998, 40000, 0, 5, 9]), cumulativeCounterModulus: 65536), 2 + 5 + 4); }); test('a reset after a LONG unsynced gap still fails the budget', () { // The gap is what buys budget, so it is capped: without the cap a 6-hour // hole would license 108000 steps and a reset would look like a wrap. final s = _sub([40000, 0, 4], startTs: 1_700_000_000, step: 21600); - expect(hardwareStepsFromCounter(s), 4); + expect(hardwareStepsFromCounter(s, cumulativeCounterModulus: 65536), 4); }); test('an implausible forward jump is dropped, not credited', () { // The per-step budget floors at 300 (the counter's update cadence is not // verified on hardware, so bursty reporting must survive); 9000 steps // between two 1 Hz records is a decode artefact, not a sprint. - expect(hardwareStepsFromCounter(_sub([100, 9100, 9104])), 4); + expect(hardwareStepsFromCounter(_sub([100, 9100, 9104]), cumulativeCounterModulus: 65536), 4); }); test('records with no counter are skipped without breaking the chain', () { // A mixed page (some records decoded without the field) must not restart // the accumulation or double-count across the hole. - expect(hardwareStepsFromCounter(_sub([10, -1, -1, 16, 18])), 8); + expect(hardwareStepsFromCounter(_sub([10, -1, -1, 16, 18]), cumulativeCounterModulus: 65536), 8); }); test('a total is never negative under any counter behaviour', () { final adversarial = [65535, 0, 65535, 0, 12, 3, 60000, 1, 1, 65000]; - final steps = hardwareStepsFromCounter(_sub(adversarial)); + final steps = hardwareStepsFromCounter(_sub(adversarial), + cumulativeCounterModulus: 65536); expect(steps, isNotNull); expect(steps!, isNonNegative); }); @@ -99,19 +100,19 @@ void main() { final cut = s.slice(1002, 1004); expect(cut.tsSec, [1002, 1003]); expect(cut.stepCount, [3, 4]); - expect(hardwareStepsFromCounter(cut), 1); + expect(hardwareStepsFromCounter(cut, cumulativeCounterModulus: 65536), 1); }); test('an absent counter round-trips as ABSENT, never as 0', () { final back = Substrate.fromJson(_sub([-1, -1, -1]).toJson()); expect(back.stepCounterAt(0), isNull); - expect(hardwareStepsFromCounter(back), isNull); + expect(hardwareStepsFromCounter(back, cumulativeCounterModulus: 65536), isNull); }); test('a present counter round-trips by value', () { final back = Substrate.fromJson(_sub([7, 9, 9, 20]).toJson()); expect(back.stepCount, [7, 9, 9, 20]); - expect(hardwareStepsFromCounter(back), 13); + expect(hardwareStepsFromCounter(back, cumulativeCounterModulus: 65536), 13); }); test('a legacy substrate with no step array reads ABSENT everywhere', () { @@ -132,7 +133,7 @@ void main() { skinContact: [0, 0, 0], ); expect(legacy.stepCounterAt(0), isNull); - expect(hardwareStepsFromCounter(legacy), isNull); + expect(hardwareStepsFromCounter(legacy, cumulativeCounterModulus: 65536), isNull); expect(legacy.slice(1, 3).stepCount, isEmpty); }); }); diff --git a/test/bandagnostic_c10_c15_test.dart b/test/bandagnostic_c10_c15_test.dart new file mode 100644 index 00000000..4d0621ea --- /dev/null +++ b/test/bandagnostic_c10_c15_test.dart @@ -0,0 +1,206 @@ +// BANDAGNOSTIC group C, wave 2 — the five changes that stop a band id or a +// gen4 constant standing in for evidence. One group per item; every case is +// pure (no DB, no BLE), because every rule under test is. + +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/live_coverage_policy.dart'; +import 'package:flutter_test/flutter_test.dart'; + +Substrate _sub({ + required List hr, + List? ax, + List? ay, + List? az, + List stepCount = const [], + String? family, +}) { + final n = hr.length; + return Substrate( + tsSec: [for (var i = 0; i < n; i++) 1_700_000_000 + i], + hr: hr, + rrTsMs: const [], + rrMs: const [], + ax: ax ?? List.filled(n, 0.0), + ay: ay ?? List.filled(n, 0.0), + az: az ?? List.filled(n, 1.0), + spo2Red: List.filled(n, 0), + spo2Ir: List.filled(n, 0), + skinTemp: List.filled(n, 0), + skinContact: List.filled(n, 0), + stepCount: stepCount, + deviceFamily: family, + ); +} + +void main() { + group('C11 — a plausibility bound refuses a reading, it does not replace it', + () { + test('the HR window is human physiology, so it is band-independent', () { + // Not a sensor property: nothing about the strap moves 25 or 230, which + // is why this does not go through calibrationFor and an unstamped record + // still gets it. + expect(plausibleHrOrNull(24), isNull); + expect(plausibleHrOrNull(25), 25); + expect(plausibleHrOrNull(230), 230); + expect(plausibleHrOrNull(231), isNull); + }); + + test('R-R is bounded as an INTERVAL, not as a rate', () { + expect(plausibleRrOrNull(249), isNull); + expect(plausibleRrOrNull(250), 250.0); + // 2,400 ms is 25 bpm only if you read one gap as a rate. One dropped + // beat doubles a gap without the heart doing anything unusual, and the + // ectopic filters downstream are built to see exactly this. + expect(plausibleRrOrNull(2400), 2400.0); + expect(plausibleRrOrNull(2401), isNull); + expect(plausibleRrOrNull(0), isNull); + expect(plausibleRrOrNull(-800), isNull); + }); + + test('accel is bounded on what a wrist SUSTAINS, not on the FSR', () { + // Exact zero is a fill, not a reading. + expect(accelPlausible(0, 0, 0), isFalse); + // Ordinary wear and real movement both pass, including well past + // protocol's gravity window (magSq 0.25..3.24) that 539a97b had to drop + // from the gen5 decoder for rejecting real workout seconds. + expect(accelPlausible(0, 0, 1.0), isTrue); + expect(accelPlausible(0.1, 0.2, 0.95), isTrue); + expect(accelPlausible(2.0, 2.0, 1.0), isTrue); // |a| = 3 g + // A second-long mean above 4 g is not a wrist. + expect(accelPlausible(0, 0, 4.0), isTrue); + expect(accelPlausible(0, 0, 4.01), isFalse); + // THE POINT OF NOT USING THE FSR: a decoder that is off by 10x on scale + // reads as absent instead of as violent movement. A +/-16 g test would + // have passed this. + expect(accelPlausible(0, 0, 10.0), isFalse); + }); + + test('an implausible second is absent, and absent is not "not worn"', () { + // A gravity vector out of range no longer counts as a present sample, so + // van Hees cannot score it as perfect immobility. + final s = _sub( + hr: const [60, 60, 60], + ax: const [0, 0, 0], + ay: const [0, 0, 0], + az: const [1.0, 0.0, 9.9], + ); + expect(s.accelPresentAt(0), isTrue); + expect(s.accelPresentAt(1), isFalse); // all-zero fill + expect(s.accelPresentAt(2), isFalse); // impossible magnitude + expect(s.accelPresentFraction(0, 3), closeTo(1 / 3, 1e-12)); + }); + }); + + group('C12 — the flag is the row\'s own, not the badge\'s', () { + test('a real flag is believed whatever the stamp says', () { + for (final family in [null, 'gen5', 'gen6', '']) { + final s = Substrate( + tsSec: const [1, 2], + hr: const [60, 60], + rrTsMs: const [], + rrMs: const [], + ax: const [0.0, 0.0], + ay: const [0.0, 0.0], + az: const [1.0, 1.0], + spo2Red: const [0, 0], + spo2Ir: const [0, 0], + skinTemp: const [0, 0], + skinContact: const [0, 0], + hrValid: const [1, 0], + deviceFamily: family, + ); + expect(s.hrValidAt(0), isTrue, reason: 'family=$family'); + expect(s.hrValidAt(1), isFalse, reason: 'family=$family'); + } + }); + }); + + group('C13 — an undeclared counter gets no number', () { + List counters(List v) => v; + + test('abstains rather than assume the counter never resets', () { + final s = _sub( + hr: List.filled(4, 60), + stepCount: counters([4200, 4260, 4300, 4340]), + ); + // The whole failure this replaces: sum-of-deltas over a MIDNIGHT-RESET + // counter publishes 140 for a morning that started at 4,200, at tier + // HIGH. Undeclared behaviour ⇒ no claim. + expect( + hardwareStepsFromCounter(s, cumulativeCounterModulus: null), + isNull, + ); + // Declared ⇒ the deltas, exactly as before. + expect( + hardwareStepsFromCounter(s, cumulativeCounterModulus: 65536), + 140, + ); + }); + + test('the modulus is the source\'s, not a constant in the function', () { + // A u8 counter wrapping 250 -> 10 is a delta of 16, and only a caller + // that declares 256 can say so. Read as u16 it is 65,296 and dropped. + final s = _sub(hr: const [60, 60], stepCount: counters([250, 10])); + expect(hardwareStepsFromCounter(s, cumulativeCounterModulus: 256), 16); + expect(hardwareStepsFromCounter(s, cumulativeCounterModulus: 65536), 0); + }); + }); + + group('C15 — equal rank competes ACROSS devices, sums within one', () { + CoverageSpan span(int from, int to, int steps, + {bool band = true, String id = ''}) => + CoverageSpan( + startTs: from, + endTs: to, + steps: steps, + fromBand: band, + deviceId: id, + ); + + test('two straps on one walk report the walk once', () { + // Both spans look like gait (100 spm), so both rank 2 — which is exactly + // the pair that used to skip overlap subtraction and publish 6,000. + final r = resolveDaySteps([ + span(0, 1800, 3000, id: 'A'), + span(0, 1800, 3000, id: 'B'), + ]); + expect(r.total, 3000); + }); + + test('partial overlap is pro-rated, not all-or-nothing', () { + final r = resolveDaySteps([ + span(0, 1800, 3000, id: 'A'), + span(900, 2700, 3000, id: 'B'), + ]); + // B keeps the half of its span A did not cover. + expect(r.total, 4500); + }); + + test('SAME device still sums — this is what re-import relies on', () { + final r = resolveDaySteps([ + span(0, 1800, 3000), + span(0, 1800, 3000), + ]); + expect(r.total, 6000); + }); + + test('the result does not depend on the order the rows came back in', () { + final a = span(0, 1800, 3000, id: 'A'); + final b = span(600, 2400, 2500, id: 'B'); + expect(resolveDaySteps([a, b]).total, resolveDaySteps([b, a]).total); + }); + }); + + group('C10 — the family list is open', () { + test('a band gets constants per METRIC, not per enum entry', () { + const gate = ana.hrCeilingMotionGateG; + expect(ana.calibrationFor(gate, 'gen4'), 0.10); + expect(ana.calibrationFor(gate, 'gen6'), isNull); + // The refusal note's grammar is parsed by the UI — pinned here because + // this is the one thing C10 was not allowed to move. + expect(ana.unknownFamilyNote('gen6'), 'unknown_device_family:id=gen6'); + expect(ana.unknownFamilyNote(null), 'unknown_device_family:id=none'); + }); + }); +} diff --git a/test/beat_clock_read_path_test.dart b/test/beat_clock_read_path_test.dart new file mode 100644 index 00000000..f4d1d70c --- /dev/null +++ b/test/beat_clock_read_path_test.dart @@ -0,0 +1,150 @@ +// The beat clock, end to end through the read path. +// +// `decoded_rr` carries two time columns. `rr_ts_ms` is `rec_ts * 1000` — a +// whole-second staircase that says every beat inside one record happened at the +// same millisecond. `beat_ts_ms` is where the beat actually was. Until now the +// read path took the staircase and the measured column was write-only. +// +// The contract this file pins: +// 1. The INTERVAL series does not move — same values, same order — so +// `hrvTime(nn)` with no time axis is bit-identical. If THAT moves, +// something other than the clock changed. (RMSSD/pNNx as production calls +// them, `hrvTime(nn, nnTimesMs: …)`, DO shift a little: the axis decides +// which successive pairs count as contiguous. Measured at +0.03% RMSSD / +// +0.6% pNN50 over a real 6 h block. SDNN never pairs, so it never moves.) +// 2. The TIME AXIS does move — that is the whole point. +// 3. A NULL `beat_ts_ms` (every row banked before the column existed, every +// source with no sub-second) falls back to the staircase. Never fabricated. +// 4. A beat whose second has no `decoded_onehz` row still reaches the +// substrate, on a second of its own with ABSENT sensors. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; + +const _t0 = 1780000000; + +Map _frame(int recTs) => { + 'rec_ts': recTs, + 'hr': 58, + 'ax': 0.0, + 'ay': 0.0, + 'az': 1.0, + 'spo2_red_raw': 1000, + 'spo2_ir_raw': 1000, + 'skin_temp_raw': 2000, + 'device_family': 'gen4', +}; + +Map _beat(int recTs, int index, int rrMs, {int? beatTsMs}) => { + 'rec_ts': recTs, + 'beat_index': index, + 'rr_ts_ms': recTs * 1000, + 'rr_ms': rrMs, + 'beat_ts_ms': beatTsMs, +}; + +/// A minute of records with two beats each, placed by the strap's own +/// sub-second anchor — which drifts, the way a real 32 kHz RTC does, so the +/// gap between one record's last beat and the next record's first is a real +/// sub-second quantity and not a multiple of 1,000 ms. +({List> frames, List> beats}) +_night() { + final frames = >[]; + final beats = >[]; + for (var s = 0; s < 60; s++) { + final recTs = _t0 + s; + frames.add(_frame(recTs)); + // The record's own sub-second (32 kHz RTC), drifting the way a real one + // does rather than sitting on the whole second. + final anchorMs = recTs * 1000 + (s * 37) % 1000; + // Two beats: the last at the anchor, the earlier one its interval before. + final rrLate = 900 + (s % 5) * 10; + final rrEarly = 920 + (s % 3) * 10; + beats.add(_beat(recTs, 0, rrEarly, beatTsMs: anchorMs - rrLate)); + beats.add(_beat(recTs, 1, rrLate, beatTsMs: anchorMs)); + } + return (frames: frames, beats: beats); +} + +void main() { + test('beat_ts_ms reaches the substrate and rr_ms is untouched', () { + final n = _night(); + final measured = substrateFromDecodedPage(n.frames, n.beats); + final staircase = substrateFromDecodedPage(n.frames, [ + for (final b in n.beats) {...b}..['beat_ts_ms'] = null, + ]); + + // 1. INTERVALS ARE IDENTICAL — same values, same order. + expect(measured.rrMs, staircase.rrMs); + final mc = correctRr(measured.rrMs, rrTsMs: measured.rrTsMs); + final sc = correctRr(staircase.rrMs, rrTsMs: staircase.rrTsMs); + // The CLEANED interval series too — the Lipponen-Tarvainen correction + // rejects on interval shape, not on where the beat sits. + expect(mc.nn, sc.nn); + // …so time-domain HRV off the intervals alone is bit-identical. SDNN stays + // identical even WITH the axis, because it never pairs. + final mHrv = hrvTime(mc.nn), sHrv = hrvTime(sc.nn); + expect(mHrv.value?.rmssd, sHrv.value?.rmssd); + expect(mHrv.value?.pnn50, sHrv.value?.pnn50); + expect( + hrvTime(mc.nn, nnTimesMs: mc.nnTimesMs).value?.sdnn, + hrvTime(sc.nn, nnTimesMs: sc.nnTimesMs).value?.sdnn, + ); + + // 2. THE AXIS MOVED. The staircase pins both beats of a record to the same + // millisecond; the measured clock separates them by their own interval. + expect(staircase.rrTsMs[0], staircase.rrTsMs[1]); + expect( + measured.rrTsMs[1] - measured.rrTsMs[0], + measured.rrMs[1], + reason: 'the spacing IS the interval', + ); + expect(measured.rrTsMs, isNot(staircase.rrTsMs)); + + // EVERY inter-record gap is a whole second on the staircase — which is the + // reason a sub-second dropout is unrepresentable there, not merely + // undetected. On the measured clock the same gaps are real. + final crossings = [for (var r = 1; r < 60; r++) r * 2]; // first beat of r + for (final i in crossings) { + expect(staircase.rrTsMs[i] - staircase.rrTsMs[i - 1], 1000); + } + expect( + crossings + .where((i) => measured.rrTsMs[i] - measured.rrTsMs[i - 1] != 1000) + .length, + crossings.length, + reason: 'no measured crossing lands on a whole second by accident', + ); + }); + + test('a NULL beat_ts_ms falls back to rr_ts_ms — never fabricated', () { + final sub = substrateFromDecodedPage( + [_frame(_t0)], + [_beat(_t0, 0, 900), _beat(_t0, 1, 910)], + ); + expect(sub.rrTsMs, [_t0 * 1000.0, _t0 * 1000.0]); + }); + + test('a beat with no 1 Hz frame still becomes a second, sensors absent', () { + // Exactly what gen4 historical R10-lite writes: an R-R block with no + // decoded_onehz row (db.dart _queueDecodedOneHz). It used to be dropped. + final sub = substrateFromDecodedPage( + [_frame(_t0), _frame(_t0 + 2)], + [ + _beat(_t0, 0, 900, beatTsMs: _t0 * 1000 + 400), + _beat(_t0 + 1, 0, 910, beatTsMs: (_t0 + 1) * 1000 + 300), + _beat(_t0 + 2, 0, 905, beatTsMs: (_t0 + 2) * 1000 + 200), + ], + ); + expect(sub.tsSec, [_t0, _t0 + 1, _t0 + 2], reason: 'union, still ascending'); + expect(sub.rrMs.length, 3, reason: 'the orphan beat survived'); + // The beat-only second carries no measurement — the absent sentinels only. + expect(sub.accelPresentAt(1), isFalse); + expect(sub.hr[1], 0); + expect(sub.skinTemp[1], 0); + // …and the frame-backed seconds are untouched. + expect(sub.accelPresentAt(0), isTrue); + expect(sub.hr[0], 58); + }); +} diff --git a/test/beat_timestamps_test.dart b/test/beat_timestamps_test.dart index 678949d5..8fdf7e98 100644 --- a/test/beat_timestamps_test.dart +++ b/test/beat_timestamps_test.dart @@ -10,6 +10,7 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/compute/substrate.dart' show beatTimesMs; import 'package:openstrap_edge/data/db.dart'; import 'package:openstrap_edge/data/models.dart'; @@ -18,17 +19,17 @@ void main() { test('no sub-second means no beat time — never a whole second instead', () { // The state of every row written before this column existed. NULL says // "we did not keep it"; rec_ts * 1000 would say "the beat was here". - expect(LocalDb.beatTimesMs(1000, null, [800, 810]), [null, null]); + expect(beatTimesMs(1000, null, [800, 810]), [null, null]); }); test('the anchor is the strap sub-second, not a rounded second', () { // 16384 ticks = exactly half a second on the 32 kHz RTC. - expect(LocalDb.beatTimesMs(1000, 16384, [800]), [1000500]); - expect(LocalDb.beatTimesMs(1000, 0, [800]), [1000000]); + expect(beatTimesMs(1000, 16384, [800]), [1000500]); + expect(beatTimesMs(1000, 0, [800]), [1000000]); // A tick is ~30 us, so the conversion floors rather than pretending to // resolve below a millisecond. - expect(LocalDb.beatTimesMs(1000, 1, [800]), [1000000]); - expect(LocalDb.beatTimesMs(1000, 32767, [800]), [1000999]); + expect(beatTimesMs(1000, 1, [800]), [1000000]); + expect(beatTimesMs(1000, 32767, [800]), [1000999]); }); test('beats are spaced by their own intervals, walking backwards', () { @@ -36,17 +37,17 @@ void main() { // after it away. Backwards is the only direction that cannot place a beat // AFTER the moment the strap told us about it. expect( - LocalDb.beatTimesMs(1000, 0, [700, 726]), + beatTimesMs(1000, 0, [700, 726]), [1000000 - 726, 1000000], ); expect( - LocalDb.beatTimesMs(1000, 0, [650, 640, 660, 644]), + beatTimesMs(1000, 0, [650, 640, 660, 644]), [1000000 - 1944, 1000000 - 1304, 1000000 - 644, 1000000], ); }); test('the spacing IS the interval — the whole point of the column', () { - final t = LocalDb.beatTimesMs(1000, 12345, [700, 726, 690]); + final t = beatTimesMs(1000, 12345, [700, 726, 690]); expect(t[1]! - t[0]!, 726); expect(t[2]! - t[1]!, 690); }); @@ -56,13 +57,13 @@ void main() { // everything earlier in the record is unplaceable. Treating the missing // gap as zero would stack two beats on one instant and call it measured. expect( - LocalDb.beatTimesMs(1000, 0, [700, 0, 690]), + beatTimesMs(1000, 0, [700, 0, 690]), [null, 1000000 - 690, 1000000], ); }); test('an empty record produces nothing', () { - expect(LocalDb.beatTimesMs(1000, 500, const []), isEmpty); + expect(beatTimesMs(1000, 500, const []), isEmpty); }); }); diff --git a/test/ble_state_test.dart b/test/ble_state_test.dart index 7938c837..d103e5b2 100644 --- a/test/ble_state_test.dart +++ b/test/ble_state_test.dart @@ -3,6 +3,7 @@ // engine — the backoff schedule, the seq allocator, the drain stop conditions, // and the phase→legacy-string projection — none of which need a real band. +import 'dart:async'; import 'dart:math'; import 'package:flutter_test/flutter_test.dart'; @@ -686,4 +687,40 @@ void main() { expect(isTimeoutDisconnect(null), isFalse); }); }); + + group('withScanLock (process-wide scan mutex)', () { + test('a queued scan does not start until the running one finishes', () async { + // The bug this exists for: two overlapping scan bodies share ONE radio + // scanner, so the second one's `stopScan` ends the first scan early and + // the first reports "found nothing" with no error anywhere. + final order = []; + final holdA = Completer(); + final a = withScanLock(() async { + order.add('a-start'); + await holdA.future; + order.add('a-end'); + return 'a'; + }); + final b = withScanLock(() async { + order.add('b-start'); + return 'b'; + }); + await Future.delayed(Duration.zero); + expect(order, ['a-start']); // b is queued, NOT running alongside a + holdA.complete(); + expect(await a, 'a'); + expect(await b, 'b'); + expect(order, ['a-start', 'a-end', 'b-start']); + }); + + test('a scan that throws releases the lock', () async { + // A blocker (revoked permission, adapter off) throws out of the band + // scan; the next scan must still run rather than wait on a dead chain. + await expectLater( + withScanLock(() async => throw StateError('adapter off')), + throwsStateError, + ); + expect(await withScanLock(() async => 7), 7); + }); + }); } diff --git a/test/cadence_decimation_rig_test.dart b/test/cadence_decimation_rig_test.dart new file mode 100644 index 00000000..9d17ada1 --- /dev/null +++ b/test/cadence_decimation_rig_test.dart @@ -0,0 +1,504 @@ +// THE DECIMATION RIG — measure what a slower band's cadence does to a metric, +// against the only ground truth we have: the same real night at 1 Hz. +// +// The premise: OpenStrap holds real 1 Hz WHOOP data, so the true answer for any +// window-based metric is already known. Take every Nth REAL sample of that same +// data, run the metric again, and compare. A cadence-aware metric converges on +// the 1 Hz answer. Most of ours do not, and this file is where that stops being +// an assertion and becomes a number. +// +// IT DOES NOT ASSERT CORRECTNESS. Today's numbers are wrong on purpose — the +// job right now is to RECORD the baseline error so the group-C agents have +// something to diff against. The only thing asserted here is the decimator +// itself (see the pure-unit test at the bottom, which runs with no database). +// +// Source data: `OPENSTRAP_TEST_DBS=/path/export.db` — same env-var idiom as +// `db_migration_ladder_test.dart` / `db_v42_retention_and_provenance_test.dart`. +// Skipped entirely when unset, because that file is personal and can never be +// committed. The database is opened READ-ONLY and never migrated: the rig reads +// `decoded_onehz` / `decoded_rr`, which have carried those column names since +// well before the v47 re-key, so no ladder is needed and nothing is written. +// +// DECIMATION DOES NOT FABRICATE. Every cadence keeps every Nth real record and +// the RR beats attached to it — no interpolation, no bucket averaging. That is +// also the physically honest model: a 60 s band emits fewer records, it does +// not emit averages of the records it skipped. Per-metric notes below say where +// a metric's own contract would have justified averaging (none of the twelve +// declare a mean input; `relative_odi` computes its own AC/DC windows and +// `cardio_stager` its own epoch means, both from raw samples). +// +// Run: +// OPENSTRAP_TEST_DBS=~/Documents/openstrap/openstrap_export_1786730410696.db \ +// flutter test test/cadence_decimation_rig_test.dart --concurrency=1 +// +// Writes `build/cadence_baseline.json` (gitignored) and prints the same table. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_analytics/onehz.dart'; + +/// Cadences under test, seconds per record. 1 is the ground truth. +/// +/// 301 is not a typo and not decoration: the median-interval helpers in +/// `load_trimp` / `hr_zones` gate on `<= 300`, so a 300 s device is fine and a +/// 301 s device falls off a cliff. `advanced_stager` gates on `< 300`, so it +/// falls one second earlier. Both edges have to be on the table. +/// +/// 15 s is here because it is where a POSITIONAL window metric does its worst +/// damage, and it is a cadence real bands actually ship (Fitbit/Garmin publish +/// 15 s HR). `nocturnalRhr` needs 1800 positions; over an 8 h night that holds +/// down to 16 s and abstains below it. So the wrong-NUMBER band is roughly +/// 2–16 s, and the top of it — where "the lowest 30-min mean" has quietly +/// become "the whole-night mean" — is the ceiling C1 has to clear. Below that +/// band the metric already fails safe. +const _cadences = [1, 5, 15, 60, 300, 301]; + +/// One `decoded_onehz` row, exactly as stored. +class _Row { + final int recTs; + final int hr; + final double ax, ay, az; + final int red, ir, temp; + const _Row(this.recTs, this.hr, this.ax, this.ay, this.az, this.red, this.ir, + this.temp); + + /// Exact (0,0,0) is the documented "never decoded" sentinel, not a reading. + bool get accelPresent => !(ax == 0 && ay == 0 && az == 0); +} + +/// One RR beat, carrying the `rec_ts` of the record that delivered it — which +/// is what decimation acts on. A band that reports every 60 s hands over the +/// beats inside the records it sends and none of the ones it skips. +class _Beat { + final int recTs; + final double tsMs; + final double rrMs; + const _Beat(this.recTs, this.tsMs, this.rrMs); +} + +/// Keep every Nth REAL sample, phase-locked to the first row's second. +/// +/// Not a resample: no value is interpolated, averaged or invented. `everyN <= 1` +/// is the identity. Non-divisor cadences (301) are honoured exactly — the point +/// of the 301 case is that it is NOT 300. +List decimateEveryNth( + List rows, int Function(T) secOf, int everyN) { + if (everyN <= 1 || rows.isEmpty) return rows; + final t0 = secOf(rows.first); + return [ + for (final r in rows) + if ((secOf(r) - t0) % everyN == 0) r + ]; +} + +/// One measured cell of the table. +class _Cell { + final String metric; + final int cadence; + final double? value; + final double? truth; + final bool absent; + final String unit; + final String note; + const _Cell(this.metric, this.cadence, this.value, this.truth, this.absent, + this.unit, this.note); + + double? get absErr => + (value == null || truth == null) ? null : (value! - truth!).abs(); + double? get relErr => (absErr == null || truth == null || truth == 0) + ? null + : absErr! / truth!.abs(); + + Map toJson() => { + 'metric': metric, + 'cadence_s': cadence, + 'value': value, + 'truth_1hz': truth, + 'abs_err': absErr, + 'rel_err': relErr, + 'absent': absent, + 'unit': unit, + if (note.isNotEmpty) 'note': note, + }; +} + +// ── window pickers ────────────────────────────────────────────────────────── +// Both are one deterministic pass over the record, so the rig picks the same +// window every run without a hand-tuned constant in it. + +/// The [spanSec] window with the LOWEST mean valid HR. Physiologically that is +/// the night, and it needs no timezone, no staging and no threshold. +(int, int) _lowestHrWindow(List<_Row> all, int spanSec) => + _extremeHrWindow(all, spanSec, lowest: true); + +/// The [spanSec] window with the HIGHEST mean valid HR — the day's hardest +/// effort, which is what the workout metrics want. +(int, int) _highestHrWindow(List<_Row> all, int spanSec) => + _extremeHrWindow(all, spanSec, lowest: false); + +(int, int) _extremeHrWindow(List<_Row> all, int spanSec, + {required bool lowest}) { + final t0 = all.first.recTs, t1 = all.last.recTs; + final n = t1 - t0 + 1; + // Second-indexed prefix sums so every candidate window is O(1). + final sum = List.filled(n + 1, 0); + final cnt = List.filled(n + 1, 0); + final hrAt = List.filled(n, 0); + for (final r in all) { + if (r.hr > 0) hrAt[r.recTs - t0] = r.hr.toDouble(); + } + for (var i = 0; i < n; i++) { + sum[i + 1] = sum[i] + hrAt[i]; + cnt[i + 1] = cnt[i] + (hrAt[i] > 0 ? 1 : 0); + } + var bestStart = t0; + double? bestMean; + // 10-min steps: fine enough to land on the night, coarse enough to be free. + for (var s = 0; s + spanSec <= n; s += 600) { + final c = cnt[s + spanSec] - cnt[s]; + // Demand real coverage — an empty window has a mean of nothing, and a + // sparsely-covered one is not the window we mean by "the night". + if (c < spanSec * 0.8) continue; + final m = (sum[s + spanSec] - sum[s]) / c; + if (bestMean == null || (lowest ? m < bestMean : m > bestMean)) { + bestMean = m; + bestStart = t0 + s; + } + } + return (bestStart, bestStart + spanSec); +} + +// ── metric adapters ───────────────────────────────────────────────────────── +// Each returns ONE scalar plus an absent flag. Absent is a first-class result +// here: "the metric refused at this cadence" is exactly as informative as a +// wrong number, and considerably more honest. + +typedef _Probe = (double?, bool); // (scalar, absent) + +_Probe _nocturnalRhrProbe(List<_Row> rows) { + // Fed EXACTLY as the app feeds it: the surviving samples, compacted. That + // compaction is the C1 defect — `windowSamples = 1800` counts positions, so + // 1800 positions is 30 min at 1 Hz and 2.5 h at 5 s. + final m = nocturnalRhr([for (final r in rows) r.hr.toDouble()]); + return (m.value?.low30Mean, !m.present); +} + +_Probe _vanHeesProbe(List<_Row> rows) { + final m = vanHeesSleepWindow([ + for (final r in rows) + AccelSample(r.recTs * 1000.0, r.ax, r.ay, r.az, valid: r.accelPresent) + ]); + return (m.value?.sptSec.toDouble(), !m.present); +} + +_Probe _relativeOdiProbe(List<_Row> rows) { + final m = relativeOdi( + [for (final r in rows) r.red.toDouble()], + [for (final r in rows) r.ir.toDouble()], + [for (final r in rows) r.recTs.toDouble()], + ); + return (m.value?.odiPerHour, !m.present); +} + +_Probe _cardioStagerProbe(List<_Row> rows, List<_Beat> beats) { + final r = cardioStager( + [for (final x in rows) x.hr.toDouble()], + [ + for (final x in rows) + AccelSample(x.recTs * 1000.0, x.ax, x.ay, x.az, valid: x.accelPresent) + ], + rrMs: [for (final b in beats) b.rrMs], + rrTsMs: [for (final b in beats) b.tsMs], + ); + // wakePct is the Cole-Kripke spine's own output and the number a cadence + // error moves first. + return (r.base.wakePct, r.base.stages.isEmpty); +} + +_Probe _hrRecoveryProbe(List<_Row> rows) { + if (rows.isEmpty) return (null, true); + var peak = 0; + for (var i = 0; i < rows.length; i++) { + if (rows[i].hr > rows[peak].hr) peak = i; + } + final m = hrRecovery( + [for (final r in rows) r.hr], + endIndex: peak, + tsSec: [for (final r in rows) r.recTs], + ); + return (m.value?.dropBpm, !m.present); +} + +_Probe _hrZonesProbe(List<_Row> rows) { + // Total accounted seconds. At any cadence a correct implementation reports + // roughly the window span; this is the C9 headline, and 301 s is where it + // stops doing that. + final t = HeartRateZones.timeInZone( + [for (final r in rows) HrSample(r.recTs * 1000.0, r.hr.toDouble())], + HeartRateZones.zonesFromMaxHr(190, source: 'rig'), + ); + return (t?.total, t == null); +} + +_Probe _loadTrimpProbe(List<_Row> rows) { + final valid = [for (final r in rows) if (r.hr > 0) r]; + final m = trimpStrain( + [for (final r in valid) r.hr.toDouble()], + [for (final r in valid) r.recTs.toDouble()], + // Fixed anchors: the rig compares a metric against ITSELF at 1 Hz, so the + // anchors only have to be constant across cadences, not personal. + maxHr: 190, + restingHr: 50, + ); + return (m.value, !m.present); +} + +_Probe _orientationProbe(List<_Row> rows) { + final tilts = positionSeries([ + for (final r in rows) + AccelSample(r.recTs * 1000.0, r.ax, r.ay, r.az, valid: r.accelPresent) + ], epochSec: 30); + return (tilts.length.toDouble(), tilts.isEmpty); +} + +_Probe _cyclesProbe(List<_Beat> beats, int onset, int offset) { + final m = sleepCyclesMetric( + [for (final b in beats) b.rrMs], + [for (final b in beats) b.tsMs], + onset, + offset, + ); + return (m.value?.n.toDouble(), !m.present); +} + +_Probe _riivProbe(List<_Row> rows) { + final m = riivRespRate( + [for (final r in rows) r.ir.toDouble()], + [for (final r in rows) r.recTs.toDouble()], + ); + return (m.value?.brpm, !m.present); +} + +_Probe _rsaProbe(List<_Beat> beats) { + final m = rsaRespRate( + [for (final b in beats) b.rrMs], + [for (final b in beats) b.tsMs], + artifactFraction: 0.0, + ); + return (m.value?.brpm, !m.present); +} + +_Probe _advancedStagerProbe(List<_Row> rows, List<_Beat> beats) { + final sessions = AdvancedSleepStager.detectSleep( + [ + for (final r in rows) + GravTs(r.recTs, r.ax, r.ay, r.az, valid: r.accelPresent) + ], + [for (final r in rows) HrTs(r.recTs, r.hr.toDouble())], + rr: [for (final b in beats) RrTs((b.tsMs / 1000).round(), b.rrMs)], + ); + if (sessions.isEmpty) return (null, true); + var best = sessions.first; + for (final s in sessions) { + if (s.end - s.start > best.end - best.start) best = s; + } + return ((best.end - best.start).toDouble(), false); +} + +// ── reporting ─────────────────────────────────────────────────────────────── + +String _fmt(double? v) { + if (v == null) return '—'; + if (v.abs() >= 1000) return v.toStringAsFixed(0); + if (v.abs() >= 10) return v.toStringAsFixed(1); + return v.toStringAsFixed(3); +} + +String _pct(double? v) => v == null ? '—' : '${(v * 100).toStringAsFixed(1)}%'; + +String _renderTable(List<_Cell> cells) { + final b = StringBuffer(); + final head = ['metric', 'unit', 'cad', '1 Hz truth', 'value', 'abs err', + 'rel err', 'absent']; + final rows = >[ + head, + for (final c in cells) + [ + c.metric, + c.unit, + '${c.cadence}s', + _fmt(c.truth), + c.absent ? 'ABSENT' : _fmt(c.value), + _fmt(c.absErr), + _pct(c.relErr), + c.absent ? 'yes' : '', + ], + ]; + final w = List.generate(head.length, + (i) => rows.map((r) => r[i].length).reduce(math.max)); + for (var ri = 0; ri < rows.length; ri++) { + b.writeln([ + for (var i = 0; i < head.length; i++) rows[ri][i].padRight(w[i]) + ].join(' ').trimRight()); + if (ri == 0) b.writeln(List.generate(w.length, (i) => '-' * w[i]).join(' ')); + } + return b.toString(); +} + +void main() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + + // The decimator is the one piece of logic in this file that could be + // silently wrong, so it gets the one check that runs without a database. + test('decimateEveryNth takes every Nth REAL sample, phase-locked, no interp', + () { + final rows = [for (var t = 100; t < 130; t++) t]; + int at(int v) => v; + expect(decimateEveryNth(rows, at, 1), rows); + expect(decimateEveryNth(rows, at, 0), rows); + expect(decimateEveryNth(rows, at, 5), [100, 105, 110, 115, 120, 125]); + // Phase-locked to the FIRST row, not to the epoch. + expect( + decimateEveryNth(rows.sublist(3), at, 5), [103, 108, 113, 118, 123, 128]); + // Non-divisor cadence: exact, not rounded to a divisor. + expect(decimateEveryNth(rows, at, 7), [100, 107, 114, 121, 128]); + // Holes survive as holes — nothing is filled in. + expect(decimateEveryNth([100, 101, 107, 110], at, 5), [100, 110]); + expect(decimateEveryNth([], at, 5), isEmpty); + }); + + final real = (Platform.environment['OPENSTRAP_TEST_DBS'] ?? '') + .split(',') + .where((s) => s.trim().isNotEmpty) + .toList(); + + for (final src in real) { + test('cadence baseline over ${p.basename(src)}', () async { + final db = await databaseFactory.openDatabase(src, + options: OpenDatabaseOptions(readOnly: true)); + + final all = <_Row>[ + for (final r in await db.query('decoded_onehz', + columns: [ + 'rec_ts', 'hr', 'ax', 'ay', 'az', + 'spo2_red_raw', 'spo2_ir_raw', 'skin_temp_raw' + ], + orderBy: 'rec_ts')) + _Row( + r['rec_ts'] as int, + r['hr'] as int, + (r['ax'] as num).toDouble(), + (r['ay'] as num).toDouble(), + (r['az'] as num).toDouble(), + (r['spo2_red_raw'] as num).toInt(), + (r['spo2_ir_raw'] as num).toInt(), + (r['skin_temp_raw'] as num).toInt(), + ) + ]; + expect(all.length, greaterThan(1000), + reason: 'need a real 1 Hz record to decimate'); + + final (nightA, nightB) = _lowestHrWindow(all, 8 * 3600); + final (dayA, dayB) = _highestHrWindow(all, 90 * 60); + + List<_Row> slice(int a, int b) => + [for (final r in all) if (r.recTs >= a && r.recTs < b) r]; + final night = slice(nightA, nightB); + final day = slice(dayA, dayB); + + Future> beatsIn(int a, int b) async => [ + for (final r in await db.rawQuery( + 'SELECT o.rec_ts AS rec_ts, r.rr_ts_ms AS ts, r.rr_ms AS rr ' + 'FROM decoded_rr r JOIN decoded_onehz o ON o.counter = r.counter ' + 'WHERE o.rec_ts >= ? AND o.rec_ts < ? ORDER BY r.rr_ts_ms', + [a, b])) + _Beat(r['rec_ts'] as int, (r['ts'] as num).toDouble(), + (r['rr'] as num).toDouble()) + ]; + final nightBeats = await beatsIn(nightA, nightB); + await db.close(); + + // ignore: avoid_print + print('[rig] ${p.basename(src)} rows=${all.length}\n' + '[rig] night window $nightA..$nightB ' + 'rows=${night.length} beats=${nightBeats.length}\n' + '[rig] day window $dayA..$dayB rows=${day.length}'); + + final cells = <_Cell>[]; + final truth = {}; + + void run(String metric, String unit, int cadence, _Probe probe, + {String note = ''}) { + final (v, absent) = probe; + if (cadence == 1) truth[metric] = v; + cells.add( + _Cell(metric, cadence, v, truth[metric], absent, unit, note)); + } + + for (final n in _cadences) { + final nRows = decimateEveryNth(night, (r) => r.recTs, n); + final dRows = decimateEveryNth(day, (r) => r.recTs, n); + // A beat rides its record: keep the beats whose parent second survived. + final keptSec = {for (final r in nRows) r.recTs}; + final nBeats = [ + for (final b in nightBeats) + if (keptSec.contains(b.recTs)) b + ]; + + run('nocturnalRhr', 'bpm', n, _nocturnalRhrProbe(nRows)); + run('vanHees.sptSec', 's', n, _vanHeesProbe(nRows)); + run('relativeOdi.perHour', '/h', n, _relativeOdiProbe(nRows)); + run('cardioStager.wakePct', '%', n, _cardioStagerProbe(nRows, nBeats)); + run('hrRecovery.dropBpm', 'bpm', n, _hrRecoveryProbe(dRows)); + run('hrZones.totalSec', 's', n, _hrZonesProbe(dRows)); + run('loadTrimp.strain', '0-100', n, _loadTrimpProbe(dRows)); + run('orientation.epochs', 'n', n, _orientationProbe(nRows)); + run('cycles.n', 'n', n, _cyclesProbe(nBeats, nightA, nightB)); + run('respRate.riiv', 'brpm', n, _riivProbe(nRows)); + run('respRate.rsa', 'brpm', n, _rsaProbe(nBeats)); + run('advancedStager.tstSec', 's', n, _advancedStagerProbe(nRows, nBeats)); + } + + // `steps` is in the group-C list and CANNOT be measured here. The + // pedometer's cadence defect (C7) lives at 50–100 Hz; the only accel this + // project ever persists is the 1 Hz substrate (invariant 1 keeps the + // 0x2B/0x33 high-rate streams RAM-only), so there is no faster source to + // decimate DOWN from. Decimating 1 Hz cannot reach it. Recorded as an + // explicit hole rather than silently dropped from the table. + cells.add(const _Cell('steps.pedometer', 0, null, null, true, 'steps', + 'NOT MEASURABLE BY DECIMATION — needs a >=50 Hz source; the 1 Hz ' + 'substrate is the fastest thing persisted. C7 needs a synthetic ' + 'rate sweep, not this rig.')); + + final table = _renderTable(cells); + // ignore: avoid_print + print('\n$table'); + + final out = File(p.join(Directory.current.path, 'build', + 'cadence_baseline.json')); + await out.parent.create(recursive: true); + await out.writeAsString(const JsonEncoder.withIndent(' ').convert({ + 'source': p.basename(src), + 'rows': all.length, + 'night_window': [nightA, nightB], + 'day_window': [dayA, dayB], + 'cadences_s': _cadences, + 'decimation': 'every Nth real sample, phase-locked to window start; ' + 'no interpolation, no bucket averaging; RR beats ride their record', + 'cells': [for (final c in cells) c.toJson()], + })); + // ignore: avoid_print + print('[rig] wrote ${out.path}'); + + // NO correctness assertions. Only that the rig produced a full grid — + // if a probe starts throwing, this is what notices. + expect(cells.length, _cadences.length * 12 + 1); + }, timeout: const Timeout(Duration(minutes: 10))); + } +} diff --git a/test/cadence_group_c_nocturnal_rig_test.dart b/test/cadence_group_c_nocturnal_rig_test.dart new file mode 100644 index 00000000..f7c2ca3b --- /dev/null +++ b/test/cadence_group_c_nocturnal_rig_test.dart @@ -0,0 +1,168 @@ +// GROUP C1 ON REAL DATA — what `nocturnalRhr` does at a slower cadence once it +// is given the clock. +// +// `cadence_decimation_rig_test.dart` measures the DEFECT: its probe feeds the +// bare HR list, which is exactly how the app feeds it today, so the cell it +// reports is the positional-window error (59.7 → 66.4 bpm at 15 s). That probe +// must keep doing that — it is the baseline everything else diffs against — so +// the fix is measured here instead, on the same night of the same export, with +// `tsSec` supplied. +// +// Run: +// OPENSTRAP_TEST_DBS=~/Documents/openstrap/openstrap_export_1786730410696.db \ +// flutter test test/cadence_group_c_nocturnal_rig_test.dart --concurrency=1 +// +// Skips cleanly with no export, and reads the night window the rig already +// picked out of `build/cadence_baseline.json` rather than re-deriving it — +// same window, no second copy of the picker. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart'; +import 'package:openstrap_edge/compute/substrate.dart' show plausibleHrOrNull; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +const _cadences = [1, 5, 15, 60, 300, 301]; + +void main() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + + final src = (Platform.environment['OPENSTRAP_TEST_DBS'] ?? '') + .split(',') + .map((s) => s.trim()) + .where((s) => s.isNotEmpty) + .toList(); + + if (src.isEmpty) { + test('C1 real-data measurement', () {}, skip: 'set OPENSTRAP_TEST_DBS'); + } + + for (final path in src) { + test('C1 nocturnalRhr with a clock, over ${p.basename(path)}', () async { + final baseline = + File(p.join(Directory.current.path, 'build', 'cadence_baseline.json')); + if (!baseline.existsSync()) { + markTestSkipped('run cadence_decimation_rig_test.dart first'); + return; + } + final meta = jsonDecode(await baseline.readAsString()) as Map; + final win = (meta['night_window'] as List).cast(); + + final db = await databaseFactory + .openDatabase(path, options: OpenDatabaseOptions(readOnly: true)); + final rows = await db.query('decoded_onehz', + columns: ['rec_ts', 'hr'], + where: 'rec_ts >= ? AND rec_ts < ?', + whereArgs: [win[0], win[1]], + orderBy: 'rec_ts'); + await db.close(); + + final ts = [for (final r in rows) (r['rec_ts'] as int).toDouble()]; + final hr = [for (final r in rows) (r['hr'] as int).toDouble()]; + expect(hr.length, greaterThan(1000)); + + double? truth; + for (final n in _cadences) { + final dts = [], dhr = []; + for (var i = 0; i < ts.length; i++) { + if ((ts[i] - ts.first) % n == 0) { + dts.add(ts[i]); + dhr.add(hr[i]); + } + } + final blind = nocturnalRhr(dhr); // what the app does today + final timed = nocturnalRhr(dhr, tsSec: dts); // what C1 adds + truth ??= timed.value?.low30Mean; + String f(double? v) => v == null ? 'ABSENT' : v.toStringAsFixed(2); + String err(double? v) => (v == null || truth == null) + ? '—' + : '${((v - truth!) / truth! * 100).toStringAsFixed(1)}%'; + // ignore: avoid_print + print('[C1] ${n.toString().padLeft(3)}s ' + 'no-ts ${f(blind.value?.low30Mean).padLeft(6)} ${err(blind.value?.low30Mean).padLeft(7)} ' + 'ts ${f(timed.value?.low30Mean).padLeft(6)} ${err(timed.value?.low30Mean).padLeft(7)} ' + 'conf ${timed.confidence.toStringAsFixed(2)}'); + + if (n == 1) { + // THE GATE: 1 Hz is bit-identical with and without the clock, and + // identical to the rig's recorded truth. + expect(timed.value!.low30Mean, blind.value!.low30Mean); + expect(timed.confidence, blind.confidence); + final cell = (meta['cells'] as List).firstWhere((c) => + c['metric'] == 'nocturnalRhr' && c['cadence_s'] == 1) as Map; + expect(timed.value!.low30Mean, cell['value']); + } else if (n <= 300) { + // Every cadence a real band ships lands on the 1 Hz trough, or says + // nothing. A number within 2% is the bar; 15 s was +11.2% before. + expect(timed.present, isTrue, reason: '${n}s: ${timed.note}'); + expect((timed.value!.low30Mean - truth!) / truth!, closeTo(0, 0.02), + reason: '${n}s'); + } else { + // Past what `sampleCadenceSeconds` will vouch for: absent, always. + expect(timed.present, isFalse, reason: '${n}s: ${timed.value}'); + } + } + }, timeout: const Timeout(Duration(minutes: 5))); + + // WHAT THE WIRING ACTUALLY MOVES. The test above decimates one night to + // show the defect at other cadences; this one asks the only question that + // decides whether shipping `tsSec` changes a published number TODAY, at + // 1 Hz, on this owner's own nights: for every real sleep window the app + // scored, is `nocturnalRhr(hr)` the same value as `nocturnalRhr(hr, tsSec)`? + // + // The windows come from `day_result.window_json` (the segmentation the app + // already ran — C1 does not move it) and the HR is read exactly as + // `derive_prepare` builds `Substrate.hr`, so the pair of numbers printed + // here is the pair the pipeline would publish. Days whose decoded rows have + // been pruned past the retention edge have no substrate to re-score and are + // skipped, which on a 3-day-retention export is most of them. + test('C1 wired: real sleep windows, blind vs timed, over ' + '${p.basename(path)}', () async { + final db = await databaseFactory + .openDatabase(path, options: OpenDatabaseOptions(readOnly: true)); + final days = await db.query('day_result', + columns: ['day_id', 'algo_version', 'rhr', 'window_json'], + orderBy: 'day_id, algo_version'); + var scored = 0; + for (final d in days) { + final w = jsonDecode((d['window_json'] as String?) ?? '{}'); + if (w is! Map || w['onset_ms'] == null || w['offset_ms'] == null) { + continue; + } + final on = ((w['onset_ms'] as num) / 1000).round(); + final off = ((w['offset_ms'] as num) / 1000).round(); + final rows = await db.query('decoded_onehz', + columns: ['rec_ts', 'hr'], + where: 'rec_ts >= ? AND rec_ts < ?', + whereArgs: [on, off], + orderBy: 'rec_ts'); + if (rows.isEmpty) continue; + scored++; + final ts = [for (final r in rows) (r['rec_ts'] as int).toDouble()]; + final hr = [ + for (final r in rows) + (plausibleHrOrNull((r['hr'] as num?)?.toInt() ?? 0) ?? 0).toDouble(), + ]; + final blind = nocturnalRhr(hr); + final timed = nocturnalRhr(hr, tsSec: ts); + String f(double? v) => v == null ? 'ABSENT' : v.toStringAsFixed(3); + final b = blind.value?.low30Mean, t = timed.value?.low30Mean; + final delta = (b == null || t == null) ? '—' : (t - b).toStringAsFixed(3); + // ignore: avoid_print + print('[C1-wired] ${d['day_id']} v${d['algo_version']} ' + 'span ${off - on}s rows ${rows.length} (holes ${off - on - rows.length}) ' + 'stored ${f((d['rhr'] as num?)?.toDouble())} ' + 'blind ${f(b)} timed ${f(t)} Δ $delta'); + } + await db.close(); + // Not an assertion about the values — this rig exists to REPORT them. The + // only thing it guards is that it actually measured something. + expect(scored, greaterThan(0), + reason: 'no scored sleep window still has decoded rows'); + }, timeout: const Timeout(Duration(minutes: 5))); + } +} diff --git a/test/db_integrity_test.dart b/test/db_integrity_test.dart index dbf510d9..27a4885d 100644 --- a/test/db_integrity_test.dart +++ b/test/db_integrity_test.dart @@ -106,12 +106,15 @@ void main() { // Written straight in: nothing in the app writes a non-NULL source to // these tables today, and that is exactly the state the filter guards. await db.insert('decoded_onehz', { + // v47 key — see _createDecodedStore. + 'ts_ms': (ts + 1) * 1000, 'counter': 701, 'rec_ts': ts + 1, 'hr': 155, 'source': 'Chest strap', }); await db.insert('decoded_rr', { + 'ts_ms': (ts + 1) * 1000, 'rec_ts': ts + 1, 'beat_index': 0, 'rr_ts_ms': (ts + 1) * 1000, @@ -144,6 +147,7 @@ void main() { // An old beat (its owning row absent — e.g. a leftover) is still deleted by // the plain rec_ts-range prune; no counter subquery, no orphan sweep needed. await db.insert('decoded_rr', { + 'ts_ms': oldTs * 1000, 'rec_ts': oldTs, 'beat_index': 0, 'rr_ts_ms': oldTs * 1000, diff --git a/test/db_migration_ladder_test.dart b/test/db_migration_ladder_test.dart index 5f164eac..1e3e998e 100644 --- a/test/db_migration_ladder_test.dart +++ b/test/db_migration_ladder_test.dart @@ -72,6 +72,33 @@ const _counterKeyedDecodedDdl = [ 'ON decoded_rr(rr_ts_ms, beat_index)', ]; +/// The PRE-v47 decoded store, exactly as a user on v43..46 has it: `rec_ts` is +/// the whole primary key of `decoded_onehz`, `decoded_rr` hangs off it by +/// `(rec_ts, beat_index)`, and `samples` is keyed by the band's flash counter. +/// This is the shape the device re-key rebuilds, and the one every 3-day +/// retention window of real 1 Hz data sits in. +const _preDeviceKeyDecodedDdl = [ + ''' + CREATE TABLE decoded_onehz ( + rec_ts INTEGER PRIMARY KEY, + counter INTEGER NOT NULL, + hr INTEGER, ax REAL, ay REAL, az REAL, + spo2_red_raw INTEGER, spo2_ir_raw INTEGER, skin_temp_raw INTEGER, + device_family TEXT, source TEXT, ts_subsec INTEGER) +''', + ''' + CREATE TABLE decoded_rr ( + rec_ts INTEGER NOT NULL, beat_index INTEGER NOT NULL, + rr_ts_ms INTEGER NOT NULL, rr_ms INTEGER NOT NULL, + device_family TEXT, source TEXT, beat_ts_ms INTEGER, + PRIMARY KEY (rec_ts, beat_index)) +''', + ''' + CREATE TABLE samples ( + counter INTEGER PRIMARY KEY, ts INTEGER NOT NULL, hr INTEGER) +''', +]; + /// The v5-era derived tables, so step 9's derived_day → day_result copy is real. const _v5DerivedDdl = [ ''' @@ -570,10 +597,17 @@ void main() { expect([for (final r in oh) r['rec_ts']], [1785000000, 1785000001, 1785000002]); expect([for (final r in oh) r['counter']], [100, 101, 102]); - // PK is now rec_ts, not counter. + // The key is no longer `counter` (v33) and, since v47, no longer rec_ts + // alone either: (device_id, ts_ms), with rec_ts demoted to the indexed + // range column and ts_ms carrying the same value it always had. final ohInfo = await db.rawQuery('PRAGMA table_info(decoded_onehz)'); - expect(ohInfo.firstWhere((c) => c['name'] == 'rec_ts')['pk'], 1); + expect(ohInfo.firstWhere((c) => c['name'] == 'device_id')['pk'], 1); + expect(ohInfo.firstWhere((c) => c['name'] == 'ts_ms')['pk'], 2); + expect(ohInfo.firstWhere((c) => c['name'] == 'rec_ts')['pk'], 0); expect(ohInfo.firstWhere((c) => c['name'] == 'counter')['pk'], 0); + expect([for (final r in oh) r['device_id']], ['', '', '']); + expect([for (final r in oh) r['ts_ms']], + [1785000000000, 1785000001000, 1785000002000]); // Beats re-home onto their real second; decoded_rr loses its counter col. final rr = @@ -849,4 +883,265 @@ void main() { expect(health['ok'], isTrue, reason: '$health'); }, ); + + test( + 'v47 re-keys decoded_onehz / decoded_rr / samples onto (device_id, ts_ms) ' + 'without losing or rewriting a row', + () async { + const name = 'migrate_v46_device_key_test.db'; + created.add(name); + await _seedOldDb( + name, + 46, + [ + ..._preDeviceKeyDecodedDdl, + ..._v5DerivedDdl, + ], + seedRows: (db) async { + for (var i = 0; i < 3; i++) { + await db.insert('decoded_onehz', { + 'rec_ts': 1786000000 + i, + 'counter': 500 + i, + 'hr': 60 + i, + 'ax': 0.1, + 'ay': 0.2, + 'az': 0.9, + 'device_family': 'gen4', + 'ts_subsec': 16384, + }); + await db.insert('samples', { + 'counter': 500 + i, + 'ts': 1786000000 + i, + 'hr': 60 + i, + }); + } + for (var b = 0; b < 2; b++) { + await db.insert('decoded_rr', { + 'rec_ts': 1786000000, + 'beat_index': b, + 'rr_ts_ms': 1786000000 * 1000, + 'rr_ms': 800 + b, + }); + } + }, + ); + + expect(await _openThroughLocalDb(name), LocalDb.schemaVersion); + final db = await LocalDb.instance; + + // NOTHING IS LOST AND NOTHING IS REWRITTEN. Same rows, same values, plus + // a key that can tell two devices apart. + final oh = await db.query('decoded_onehz', orderBy: 'rec_ts ASC'); + expect([for (final r in oh) r['rec_ts']], + [1786000000, 1786000001, 1786000002]); + expect([for (final r in oh) r['hr']], [60, 61, 62]); + expect([for (final r in oh) r['counter']], [500, 501, 502]); + // Columns added off-ladder survive the PRAGMA-derived rebuild — the whole + // reason the DDL is reconstructed instead of hardcoded. + expect([for (final r in oh) r['ts_subsec']], [16384, 16384, 16384]); + expect([for (final r in oh) r['device_family']], ['gen4', 'gen4', 'gen4']); + // '' is the primary band, reserved permanently; ts_ms is rec_ts*1000 + // EXACTLY, so the key is as unique as rec_ts was. + expect([for (final r in oh) r['device_id']], ['', '', '']); + expect([for (final r in oh) r['ts_ms']], + [1786000000000, 1786000001000, 1786000002000]); + + for (final t in const ['decoded_onehz', 'decoded_rr', 'samples']) { + final info = await db.rawQuery('PRAGMA table_info($t)'); + expect(info.firstWhere((c) => c['name'] == 'device_id')['pk'], 1, + reason: t); + expect(info.firstWhere((c) => c['name'] == 'ts_ms')['pk'], 2, + reason: t); + } + // decoded_rr's key is its parent's, one level deeper. + final rrInfo = await db.rawQuery('PRAGMA table_info(decoded_rr)'); + expect(rrInfo.firstWhere((c) => c['name'] == 'beat_index')['pk'], 3); + final rr = + await db.query('decoded_rr', orderBy: 'ts_ms ASC, beat_index ASC'); + expect([for (final r in rr) r['rr_ms']], [800, 801]); + expect([for (final r in rr) r['ts_ms']], + [1786000000000, 1786000000000]); + + // `samples` was keyed by the WHOOP flash counter; it is keyed by time now + // and `counter` is demoted to a plain column. + final sm = await db.query('samples', orderBy: 'ts ASC'); + expect([for (final r in sm) r['ts_ms']], + [1786000000000, 1786000001000, 1786000002000]); + expect([for (final r in sm) r['counter']], [500, 501, 502]); + + // rec_ts stops being the key, so it MUST become an index — every read in + // db.dart ranges over it and would otherwise scan the whole table. + final ix = await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type = 'index' " + "AND tbl_name IN ('decoded_onehz', 'decoded_rr')", + ); + final ixNames = {for (final r in ix) r['name']}; + expect(ixNames, contains('idx_decoded_onehz_rects')); + expect(ixNames, contains('idx_decoded_rr_rects')); + + // No leaked temp tables. + expect( + await db.rawQuery( + "SELECT name FROM sqlite_master WHERE name LIKE '%\\_v47' ESCAPE '\\'", + ), + isEmpty, + ); + + final health = await LocalDb.schemaHealth(); + expect(health['ok'], isTrue, reason: '$health'); + }, + ); + + test( + 'THE DEDUPE REGRESSION: re-ingesting an already-migrated second leaves ' + 'EXACTLY ONE row — it fires on the next sync, not at migration time', + () async { + // The v47 re-key is only correct if `ts_ms = rec_ts * 1000` keeps the key + // exactly as unique as `rec_ts` was. Widen it by one sub-second and the + // band's post-reboot counter reset, a re-drained flash region and a + // re-delivered batch each start writing a SECOND row for a second that + // already has one — silently doubling the substrate. A ladder test cannot + // see that: it only shows up the next time the writer runs. + const name = 'v47_dedupe_test.db'; + created.add(name); + await _seedOldDb(name, 46, [..._preDeviceKeyDecodedDdl, ..._v5DerivedDdl], + seedRows: (db) async { + await db.insert('decoded_onehz', { + 'rec_ts': 1786000000, + 'counter': 900, + 'hr': 55, + 'ts_subsec': 100, + }); + await db.insert('decoded_rr', { + 'rec_ts': 1786000000, + 'beat_index': 0, + 'rr_ts_ms': 1786000000 * 1000, + 'rr_ms': 1000, + }); + }); + expect(await _openThroughLocalDb(name), LocalDb.schemaVersion); + + // The SAME second re-offloaded under a DIFFERENT counter (what a band + // reboot produces) and with a different sub-second and a shrinking beat + // set — the three things that used to fight over the key. + await LocalDb.insertRecordsBatch( + [ + RawRecord( + counter: 7, + packetType: 47, + hex: _v24RecordHex( + counter: 7, + tsEpoch: 1786000000, + hr: 58, + rrMs: 950, + ), + capturedAt: 1786000000 * 1000, + recTs: 1786000000, + ), + ], + [null], + ); + + final db = await LocalDb.instance; + final oh = await db.query('decoded_onehz'); + expect(oh.length, 1, reason: 'newest-wins dedupe must survive the re-key'); + expect(oh.first['hr'], 58, reason: 'the fresher offload wins'); + expect(oh.first['ts_ms'], 1786000000000); + // The beat set is REPLACED, not merged — and the delete that does it is + // now scoped to the writing device. + final rr = await db.query('decoded_rr'); + expect(rr.length, 1); + expect(rr.first['rr_ms'], 950); + }, + ); + + test( + 'THE POINT OF THE WHOLE PHASE: a primary row and a secondary row at the ' + 'SAME instant both survive', + () async { + // Before v47 this was impossible by construction: `decoded_onehz` was + // `rec_ts INTEGER PRIMARY KEY` written with REPLACE and `decoded_rr` was + // cleared by an unscoped `DELETE ... WHERE rec_ts = ?`, so the second + // device did not merge with the first — it DELETED it, row and beats, and + // raw_archive prunes at 3 days. + const name = 'v47_two_devices_test.db'; + created.add(name); + await _seedOldDb(name, 46, [..._preDeviceKeyDecodedDdl, ..._v5DerivedDdl], + seedRows: (db) async { + await db.insert('decoded_onehz', + {'rec_ts': 1786000000, 'counter': 1, 'hr': 61}); + await db.insert('decoded_rr', { + 'rec_ts': 1786000000, + 'beat_index': 0, + 'rr_ts_ms': 1786000000 * 1000, + 'rr_ms': 980, + }); + }); + expect(await _openThroughLocalDb(name), LocalDb.schemaVersion); + final db = await LocalDb.instance; + + await db.insert( + 'decoded_onehz', + { + 'device_id': 'polar-h10:AABBCC', + 'ts_ms': 1786000000 * 1000, + 'rec_ts': 1786000000, + 'counter': 1, + 'hr': 59, + 'source': 'polar_h10', + }, + conflictAlgorithm: ConflictAlgorithm.replace, + ); + await db.insert( + 'decoded_rr', + { + 'device_id': 'polar-h10:AABBCC', + 'ts_ms': 1786000000 * 1000, + 'rec_ts': 1786000000, + 'beat_index': 0, + 'rr_ts_ms': 1786000000 * 1000, + 'rr_ms': 1010, + 'source': 'polar_h10', + }, + conflictAlgorithm: ConflictAlgorithm.replace, + ); + + final oh = await db.query('decoded_onehz', orderBy: 'device_id ASC'); + expect(oh.length, 2); + expect([for (final r in oh) r['device_id']], ['', 'polar-h10:AABBCC']); + expect([for (final r in oh) r['hr']], [61, 59]); + + final rr = await db.query('decoded_rr', orderBy: 'device_id ASC'); + expect(rr.length, 2); + expect([for (final r in rr) r['rr_ms']], [980, 1010]); + + // And the primary band writing that same second again clears only ITS OWN + // beats — the unscoped delete is what made a second device destructive. + await LocalDb.insertRecordsBatch( + [ + RawRecord( + counter: 2, + packetType: 47, + hex: _v24RecordHex( + counter: 2, + tsEpoch: 1786000000, + hr: 62, + rrMs: 970, + ), + capturedAt: 1786000000 * 1000, + recTs: 1786000000, + ), + ], + [null], + ); + final after = await db.query('decoded_rr', orderBy: 'device_id ASC'); + expect(after.length, 2); + expect([for (final r in after) r['rr_ms']], [970, 1010]); + expect( + (await db.query('decoded_onehz')).length, + 2, + reason: 'the strap must never evict the other device', + ); + }, + ); } diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart index 21a8d530..ab2584b5 100644 --- a/test/db_serve_version_and_reads_test.dart +++ b/test/db_serve_version_and_reads_test.dart @@ -180,6 +180,8 @@ void main() { final end = localDayEndSec(label)!; for (final ts in [start + 60, end - 90]) { await db.insert('decoded_onehz', { + // v47 key — see _createDecodedStore. + 'ts_ms': ts * 1000, 'rec_ts': ts, 'counter': ts, 'hr': 60, diff --git a/test/db_storage_hygiene_test.dart b/test/db_storage_hygiene_test.dart index 16e980d5..c3999944 100644 --- a/test/db_storage_hygiene_test.dart +++ b/test/db_storage_hygiene_test.dart @@ -1,6 +1,7 @@ // Storage hygiene: -// 1. decoded_rr is keyed (rec_ts, beat_index) and carries no redundant -// secondary index; rec_ts-range reads are served by the PK auto-index. +// 1. decoded_rr is keyed (device_id, ts_ms, beat_index) since v47, so the +// rec_ts-range derive read is served by ONE secondary index on +// (rec_ts, beat_index) — seek, not scan, and no sort. // 2. Superseded generations of the recomputable per-day intermediates are // pruned. They are keyed (day_id, algo_version), so every kAlgoVersion // bump wrote a whole new generation beside the old one and nothing @@ -27,21 +28,25 @@ void main() { await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); }); - test('decoded_rr carries no redundant secondary index', () async { + test('decoded_rr carries exactly ONE secondary index', () async { final db = await LocalDb.instance; final idx = await _rrIndexes(db); - // The (rec_ts, beat_index) PRIMARY KEY auto-indexes; nothing else should be - // maintained on this hot insert path. + // Since v47 the key is (device_id, ts_ms, beat_index), so the PK auto-index + // no longer starts with rec_ts and CANNOT serve the derive read path. One + // index on (rec_ts, beat_index) buys it back — and it is the ONLY one this + // hot insert path may carry. expect( idx.where((n) => !n.startsWith('sqlite_autoindex')), - isEmpty, + ['idx_decoded_rr_rects'], reason: 'unexpected secondary index on decoded_rr: $idx', ); }); - test('rec_ts-range reads on decoded_rr are served by the PK auto-index', () async { - // decoded_rr shares the rec_ts key with decoded_onehz, so the derive read - // path (decodedRrByRecTsRange) is a PK range scan — never a full-table read. + test('rec_ts-range reads on decoded_rr seek, and sort from the index', + () async { + // decoded_rr still shares its RANGE key with decoded_onehz, so the derive + // read path (decodedRrByRecTsRange) is an index range scan — never a + // full-table read, and never a sort. final db = await LocalDb.instance; final detail = (await db.rawQuery( 'EXPLAIN QUERY PLAN SELECT * FROM decoded_rr WHERE rec_ts BETWEEN 1 AND 9 ' @@ -60,7 +65,7 @@ void main() { expect( plan, isNot(contains('USE TEMP B-TREE')), - reason: 'ordering should come from the PK: $detail', + reason: 'ordering should come from the index: $detail', ); }); diff --git a/test/db_v42_retention_and_provenance_test.dart b/test/db_v42_retention_and_provenance_test.dart index 10e15e85..4a355fd2 100644 --- a/test/db_v42_retention_and_provenance_test.dart +++ b/test/db_v42_retention_and_provenance_test.dart @@ -187,6 +187,66 @@ void main() { expect(rows.last['source'], isNull); }); + test('a strap swap is masked PER-METRIC, and only where the units differ', + () async { + await _useFreshDb('family_seam_test.db'); + // Three gen4 nights, then the athlete swaps to a gen5. `skin_temp_adc` is + // ADC COUNTS on one side of that seam and CENTI-DEGREES on the other, + // under one key, feeding one baseline — this is live today with no second + // device involved. + for (final d in const [ + ('2026-03-01', 'gen4'), + ('2026-03-02', 'gen4'), + ('2026-03-03', 'gen5'), + ]) { + await LocalDb.putDayResult( + dayId: d.$1, + algoVersion: 76, + payloadJson: '{}', + windowJson: '{}', + source: 'band', + deviceFamily: d.$2, + series: {'rhr': 55, 'skin_temp_adc': 30000}, + ); + } + // Foreign = not the newest stamped family. The gen5 night is the CURRENT + // one, so the two gen4 nights are what a skin-temp baseline must drop. + expect(await LocalDb.foreignFamilyDates(), {'2026-03-01', '2026-03-02'}); + // ...and only skin temp. RHR off a different WHOOP is the same quantity + // measured slightly differently — masking it would delete real history to + // fix a bias smaller than the window it is measured over. + expect(LocalDb.familySeamKeys, {'skin_temp_adc'}); + + // A day with NO stamp is UNKNOWN, never foreign — every day written before + // the column existed reads NULL, and dropping those would empty a real + // user's baseline. + await LocalDb.putDayResult( + dayId: '2026-03-04', + algoVersion: 76, + payloadJson: '{}', + windowJson: '{}', + series: {'rhr': 56}, + ); + expect(await LocalDb.foreignFamilyDates(), isNot(contains('2026-03-04'))); + }); + + test('one family is never foreign to itself — the mask moves no number today', + () async { + await _useFreshDb('family_seam_single_test.db'); + for (final d in const ['2026-04-01', '2026-04-02']) { + await LocalDb.putDayResult( + dayId: d, + algoVersion: 76, + payloadJson: '{}', + windowJson: '{}', + source: 'band', + deviceFamily: 'gen4', + series: {'skin_temp_adc': 30000}, + ); + } + expect(await LocalDb.foreignFamilyDates(), isEmpty); + }); + test('the 3-day prune keeps wear/charge transitions and drops the rest', () async { await _useFreshDb('v42_band_events_test.db'); @@ -315,6 +375,94 @@ void main() { .where((s) => s.trim().isNotEmpty) .toList(); for (final src in real) { + test( + 're-keys ${p.basename(src)} onto (device_id, ts_ms) — and how long the ' + 'launch-path ladder takes to do it', + () async { + // `onUpgrade` runs inside openDatabase, on iOS's launch-path CPU + // watchdog. The v47 rebuild is bounded by `rawRetentionDays` (~3 days + // of 1 Hz), which is what makes it safe to run there — this test is + // where that claim gets a number instead of an assurance. + final name = 'v47_${p.basenameWithoutExtension(src)}.db'; + await _useFreshDb(name); + final dir = await databaseFactory.getDatabasesPath(); + await File(src).copy(p.join(dir, name)); + + // EVERY day_result BEFORE the ladder runs. Phase 2 must not move a + // single computed number — no kAlgoVersion bump ships with it — so the + // payloads have to come back byte-identical. Opened with NO version so + // sqflite does not migrate it out from under the snapshot. + final plain = await databaseFactory.openDatabase( + p.join(dir, name), + options: OpenDatabaseOptions(readOnly: true), + ); + final beforeDays = { + for (final r in await plain.query('day_result')) + '${r['day_id']}|${r['algo_version']}': r['payload_json'], + }; + await plain.close(); + expect(beforeDays, isNotEmpty); + + final sw = Stopwatch()..start(); + final db = await LocalDb.instance; + sw.stop(); + final afterDays = { + for (final r in await db.query('day_result')) + '${r['day_id']}|${r['algo_version']}': r['payload_json'], + }; + expect(afterDays, beforeDays, + reason: 'the re-key must not touch a derived number'); + final before = {}; + for (final t in const ['decoded_onehz', 'decoded_rr', 'samples']) { + before[t] = await _count('SELECT COUNT(*) FROM $t'); + } + // ignore: avoid_print + print('[v47] ${p.basename(src)} whole ladder open: ' + '${sw.elapsedMilliseconds} ms rows=$before'); + + for (final t in const ['decoded_onehz', 'decoded_rr', 'samples']) { + final info = await db.rawQuery('PRAGMA table_info($t)'); + expect(info.firstWhere((c) => c['name'] == 'device_id')['pk'], 1, + reason: t); + expect(info.firstWhere((c) => c['name'] == 'ts_ms')['pk'], 2, + reason: t); + // Every migrated row belongs to the primary band and carries the + // time it always had. Nothing is rewritten, nothing is dropped. + expect( + await _count("SELECT COUNT(*) FROM $t WHERE device_id <> ''"), + 0, + reason: t, + ); + } + expect( + await _count( + 'SELECT COUNT(*) FROM decoded_onehz WHERE ts_ms <> rec_ts * 1000', + ), + 0, + ); + expect( + await _count( + 'SELECT COUNT(*) FROM decoded_rr WHERE ts_ms <> rec_ts * 1000', + ), + 0, + ); + expect(await _count('SELECT COUNT(*) FROM samples WHERE ts_ms <> ts * 1000'), 0); + + // Re-opening is a no-op: the rung self-skips on a table that already + // carries device_id, so a second launch pays nothing. + await LocalDb.close(); + final sw2 = Stopwatch()..start(); + await LocalDb.instance; + sw2.stop(); + // ignore: avoid_print + print('[v47] ${p.basename(src)} second open (no ladder): ' + '${sw2.elapsedMilliseconds} ms'); + + final health = await LocalDb.schemaHealth(); + expect(health['ok'], isTrue, reason: '$src $health'); + }, + timeout: const Timeout(Duration(minutes: 15)), + ); test( 'migrates ${p.basename(src)} to v42', () async { diff --git a/test/db_v43_nullable_hr_test.dart b/test/db_v43_nullable_hr_test.dart index 9afe59a2..92f2d896 100644 --- a/test/db_v43_nullable_hr_test.dart +++ b/test/db_v43_nullable_hr_test.dart @@ -116,7 +116,10 @@ void main() { // always used for "no heart rate this second" — never as a throw. final samples = await LocalDb.samplesInRange(2000, 2002); expect(samples.map((s) => s.hr), [60, 0, 80]); - expect(samples[1].wristOn, isFalse); + // `Sample.wristOn` (`hr > 0`) is GONE — BANDAGNOSTIC C11. It was the one + // reader in the app that turned "no usable heart rate this second" into + // "the band was off your wrist", and it had no callers. Wear truth is the + // HELLO body, the wrist on/off events and record presence. // 2. Every SQL read is gated `hr > 0`, which NULL fails — so the // heart-rate readers simply do not see the second, rather than diff --git a/test/night_beats_repo_test.dart b/test/night_beats_repo_test.dart index 36b58b3f..44d2534f 100644 --- a/test/night_beats_repo_test.dart +++ b/test/night_beats_repo_test.dart @@ -43,6 +43,9 @@ Future _seedBeats(Database db, int fromSec, int toSec) async { final batch = db.batch(); for (var t = fromSec; t <= toSec; t++) { batch.insert('decoded_rr', { + // v47: the key is (device_id, ts_ms, beat_index). A hand-written row that + // omits ts_ms takes the DEFAULT 0 and collides with every other one. + 'ts_ms': t * 1000, 'rec_ts': t, 'beat_index': 0, 'rr_ts_ms': t * 1000, diff --git a/test/observation_isolation_test.dart b/test/observation_isolation_test.dart new file mode 100644 index 00000000..03ba779f --- /dev/null +++ b/test/observation_isolation_test.dart @@ -0,0 +1,492 @@ +// THE INVARIANT that the `observation` table exists to have. +// +// OBSERVATION_SPEC §3: *nothing reads `observation` into a baseline, into a +// trend that also contains derived values, or into any input to a derivation.* +// +// The table is easy. The guarantee is the work, because a violation of it is +// SILENT — no exception, no wrong-looking row, just an unexplained step change +// in a 28- or 90-day number that surfaces months later with no way to tell +// which day broke it. This project has re-encountered that exact failure mode +// (the duplicate-day baseline pollution behind the blank readiness ring), and +// the lesson banked from it was that a comment saying "don't" is not a control. +// +// Three layers, each reusing a mechanism this repo already has: +// +// 1. STRUCTURAL — the table name may only be reached from `lib/data/db.dart`. +// Uses the same source scan the other structural tests are built on. This +// is the layer that catches the realistic violation: phase 4 adds a reader +// for the UI, and six months later someone joins it into a series. +// 2. DIFFERENTIAL — every baseline / trend / derivation read in the app is +// snapshotted, poison observations are written on the SAME dates under the +// SAME key names with absurd values, and every read must come back +// byte-identical. This is the layer that catches a violation added INSIDE +// db.dart, where layer 1 cannot see. +// 3. THE COACH BTREE GATE — already fail-closed on anything outside the coach +// views, so it needs no code. Asserted here so that adding `observation` +// to a `v_*` view (the one edit that would open it to an LLM prompt) goes +// red instead of shipping. + +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/coach/coach_db.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/observation.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider_platform_interface/path_provider_platform_interface.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +/// `exportCopy` writes a real file, so the backup round trip needs somewhere +/// to put it. +class _FakePathProvider extends PathProviderPlatform { + _FakePathProvider(this.root); + final String root; + @override + Future getTemporaryPath() async => root; + @override + Future getApplicationSupportPath() async => root; + @override + Future getApplicationDocumentsPath() async => root; +} + +/// The ONLY files allowed to name the `observation` table. +/// +/// Adding to this list is the deliberate act the invariant asks for. Before +/// you do: a READER for the UI is fine (§6 — show their number, attributed). +/// A reader that feeds `metric_series`, `day_result`, a `baselines` row, a +/// rolling window, or any argument to an analytics function is the thing this +/// whole file exists to stop. +const _allowedToNameTheTable = {'lib/data/db.dart'}; + +/// How a Dart file actually reaches a SQLite table: as a quoted table name +/// handed to sqflite, or named in SQL text. Case-insensitive on the keyword, +/// case-SENSITIVE on the table name so `ui2/grammar.dart`'s `Observation` +/// widget (a different thing that is correctly named the same word) is not a +/// match. +/// +/// `'observation':` is excluded — a trailing colon is a Dart MAP KEY and never +/// a table name. `ui2/profile/gallery.dart` keys its widget catalogue that way. +final _reachesTable = RegExp( + r"""(?:['"]observation['"](?!\s*:))""" + r"""|(?:\b(?:from|join|into|update|table)\s+observation\b)""", + caseSensitive: false, +); + +/// `observation` is lower-case in every table-name position; the widget is not. +bool _namesTheTable(String line) => + _reachesTable.hasMatch(line) && line.contains('observation'); + +/// A line that is nothing but a comment cannot reach a table. +final _pureComment = RegExp(r'^\s*(///|//|\*|/\*)'); + +Observation _obs( + String iso, + String key, + double v, { + ObservationSource kind = ObservationSource.vendor, +}) => Observation( + key: key, + value: v, + unit: 'x', + attribution: 'Amazfit', + at: DateTime.parse(iso), + sourceKind: kind, +); + +Future _measured(String date, double rhr, double lnRmssd) => + LocalDb.putDayResult( + dayId: date, + algoVersion: 1, + payloadJson: '{"date":"$date"}', + windowJson: '{}', + finalized: true, + source: 'band', + rhr: rhr, + series: {'rhr': rhr, 'ln_rmssd': lnRmssd, 'readiness': 60}, + ); + +/// Every read in the app that a baseline, a trend or a derivation runs on. +/// Rendered to a single string so a difference of any kind fails. +Future _everyDerivedRead() async { + final db = await LocalDb.instance; + final out = StringBuffer(); + for (final key in const ['rhr', 'ln_rmssd', 'readiness', 'skin_temp_adc']) { + // The production rolling-baseline loader itself, not a stand-in. + out.writeln('window($key)=${await debugBaselineWindow(key)}'); + out.writeln('trail($key)=${await LocalDb.trailingSeriesValues(key, 90)}'); + out.writeln( + 'trailAll($key)=' + '${await LocalDb.trailingSeriesValues(key, 90, measuredOnly: false)}', + ); + out.writeln('series($key)=${await LocalDb.metricSeries(key)}'); + out.writeln( + 'seriesMeasured($key)=' + '${await LocalDb.metricSeries(key, measuredOnly: true)}', + ); + out.writeln('baseline($key)=${await LocalDb.baseline(key)}'); + } + out.writeln('importedDates=${(await LocalDb.importedDates()).toList()..sort()}'); + // The frozen day bundles: a vendor scalar reaching one is spec §7's first + // "must never happen". + for (final r in await db.query('day_result', orderBy: 'day_id')) { + out.writeln('day ${r['day_id']}|${r['algo_version']}=${r['payload_json']}'); + } + for (final r in await db.query('metric_series', orderBy: 'date, key')) { + out.writeln('ms ${r['date']}|${r['key']}=${r['value']}'); + } + return out.toString(); +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + // ── LAYER 1 — structural ──────────────────────────────────────────────────── + test('only lib/data/db.dart may reach the observation table', () { + final offenders = []; + for (final e in Directory('lib').listSync(recursive: true)) { + if (e is! File || !e.path.endsWith('.dart')) continue; + final rel = p.relative(e.path).replaceAll(r'\', '/'); + if (_allowedToNameTheTable.contains(rel)) continue; + final lines = e.readAsLinesSync(); + for (var i = 0; i < lines.length; i++) { + if (_pureComment.hasMatch(lines[i])) continue; + if (_namesTheTable(lines[i])) offenders.add('$rel:${i + 1}'); + } + } + expect( + offenders, + isEmpty, + reason: + 'A vendor/typed-in/imported scalar must never reach a baseline, a ' + 'mixed trend, or a derivation input (OBSERVATION_SPEC §3). If this ' + 'is a DISPLAY reader, add the file to _allowedToNameTheTable and say ' + 'in its doc comment that nothing computed reads it. If it feeds a ' + 'number we compute, it is the bug this test exists to catch.', + ); + }); + + test('the scanner would actually catch a violation', () { + // A guard whose primitive is broken fails green. These are the shapes a + // real violation takes. + for (final line in const [ + " final rows = await db.query('observation');", + ' "SELECT value FROM observation WHERE key = ?",', + " 'JOIN observation o ON o.date = m.date '", + r''' "INSERT INTO observation (key) VALUES (?)",''', + ]) { + expect(_namesTheTable(line), isTrue, reason: line); + } + // …and the shapes that are not. + for (final line in const [ + ' const Observation(headline, detail),', + '/// One battery observation from `band_battery`.', + ' class Observation extends StatelessWidget {', + // The widget catalogue's map key — a Dart map key, not a table. + " 'observation': const Observation(", + ]) { + expect( + _pureComment.hasMatch(line) || !_namesTheTable(line), + isTrue, + reason: line, + ); + } + }); + + group('runtime', () { + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + PathProviderPlatform.instance = _FakePathProvider( + (await Directory.systemTemp.createTemp('openstrap_obs_')).path, + ); + LocalDb.dbName = 'openstrap_observation_test.db'; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + }); + + tearDownAll(() async { + await CoachDb.close(); + await LocalDb.close(); + }); + + // ── LAYER 2 — differential ─────────────────────────────────────────────── + test('observations move NO derived number, anywhere', () async { + await _measured('2026-01-01', 50, 4.0); + await _measured('2026-01-02', 52, 4.2); + await _measured('2026-01-03', 51, 4.1); + await LocalDb.putBaseline('rhr', '{"median":51}'); + + final before = await _everyDerivedRead(); + expect(before, contains('window(rhr)=[50.0, 52.0, 51.0]')); + + // Poison: OUR key names, THOSE days, values no baseline could survive, + // one of every source_kind. If any read below admits one of these, the + // window/median/trend moves and the snapshot stops matching. + expect( + await LocalDb.putObservations([ + _obs('2026-01-01T09:00:00Z', 'rhr', 999), + _obs('2026-01-02T09:00:00Z', 'rhr', -999), + _obs('2026-01-03T09:00:00Z', 'rhr', 999), + _obs('2026-01-01T09:00:00Z', 'ln_rmssd', 999), + _obs( + '2026-01-02T09:00:00Z', + 'readiness', + 0, + kind: ObservationSource.imported, + ), + _obs( + '2026-01-03T09:00:00Z', + 'skin_temp_adc', + 999, + kind: ObservationSource.entered, + ), + ]), + 6, + ); + final db = await LocalDb.instance; + expect( + ((await db.rawQuery('SELECT COUNT(*) c FROM observation')).first['c'] + as num) + .toInt(), + 6, + ); + + expect( + await _everyDerivedRead(), + before, + reason: + 'Something now reads `observation` into a computed number. That is ' + 'OBSERVATION_SPEC §3, and the failure it causes in production is ' + 'silent — an unexplained step change in a long-horizon metric.', + ); + }); + + // ── LAYER 3 — the coach's read surface ─────────────────────────────────── + test('the coach btree gate cannot reach observations', () async { + for (final sql in const [ + 'SELECT * FROM observation LIMIT 50', + 'SELECT value FROM v_metric UNION ALL SELECT value FROM observation', + "SELECT * FROM v_daily WHERE date IN (SELECT date FROM observation)", + ]) { + await expectLater( + CoachDb.debugAssertAllowedBtrees(sql), + throwsA(isA()), + reason: 'an LLM prompt reached the observation store: $sql', + ); + } + }); + + // ── the storage layer itself ───────────────────────────────────────────── + test('identity is COALESCE(vendor_key, key) and REPLACE actually replaces', + () async { + final db = await LocalDb.instance; + await db.delete('observation'); + final at = DateTime.parse('2026-02-01T08:00:00Z'); + Observation biocharge(double v) => Observation( + vendorKey: 'BioCharge', + value: v, + attribution: 'Amazfit', + at: at, + sourceKind: ObservationSource.vendor, + ); + + // THE TRAP a plain composite PRIMARY KEY falls into: `key` is NULL on a + // proprietary composite, SQLite treats NULLs in a UNIQUE index as + // DISTINCT, so the second write would not collide and a re-import would + // silently double every composite it carries. The expression index has + // no NULL to be distinct about. + await LocalDb.putObservation(biocharge(40)); + await LocalDb.putObservation(biocharge(70)); + final rows = await db.query('observation'); + expect(rows, hasLength(1)); + expect(rows.single['value'], 70.0); + expect(rows.single['vendor_key'], 'BioCharge'); + expect(rows.single['key'], isNull); + // The primary band, permanently — same standing rule as decoded_onehz. + expect(rows.single['device_id'], ''); + + // Our-vocabulary rows key off `key` instead, and do not collide with it. + await LocalDb.putObservation( + Observation( + key: 'steps', + value: 8000, + attribution: 'Amazfit', + at: at, + sourceKind: ObservationSource.vendor, + ), + ); + // Nor does the same name under a different source_kind: 'you typed 8000' + // and 'the band counted 8000' are two facts, not one. + await LocalDb.putObservation( + Observation( + key: 'steps', + value: 8000, + attribution: 'you', + at: at, + sourceKind: ObservationSource.entered, + ), + ); + expect(await db.query('observation'), hasLength(3)); + }); + + test('date is the LOCAL day label, including across a DST boundary', + () async { + final db = await LocalDb.instance; + await db.delete('observation'); + // 00:30 local on a spring-forward morning: a UTC-derived label puts this + // on the wrong day for every user west of Greenwich, which is the exact + // bug class `day_label.dart` exists to prevent. + final at = DateTime(2026, 3, 29, 0, 30); + await LocalDb.putObservation( + Observation( + key: 'mood', + value: 3, + attribution: 'you', + at: at, + sourceKind: ObservationSource.entered, + ), + ); + final row = (await db.query('observation')).single; + expect(row['date'], '2026-03-29'); + expect(row['ts_ms'], at.millisecondsSinceEpoch); + }); + + test('a row with no name at all is refused by the database', () async { + final db = await LocalDb.instance; + // Not just the Dart assert — the CHECK is what makes COALESCE non-NULL, + // and therefore what makes the unique index enforce anything. + await expectLater( + db.insert('observation', { + 'ts_ms': 1, + 'date': '2026-01-01', + 'source_kind': 'vendor', + 'value': 1.0, + 'attribution': 'Amazfit', + }), + throwsA(anything), + ); + }); + + test('the table survives a backup/restore round trip', () async { + final db = await LocalDb.instance; + await db.delete('observation'); + await LocalDb.putObservation( + _obs('2026-04-01T10:00:00Z', 'sleep_score', 82), + ); + final snapshot = await LocalDb.exportCopy(); + await db.delete('observation'); + await LocalDb.importFromDbFile(snapshot); + final rows = await db.query('observation'); + expect( + rows, + hasLength(1), + reason: 'a table absent from the merge manifest silently vanishes on ' + 'restore — nothing regenerates a vendor scalar or a typed-in one', + ); + expect(rows.single['value'], 82.0); + await File(snapshot).delete(); + }); + + test('schemaHealth requires it', () async { + final health = await LocalDb.schemaHealth(); + expect(health['ok'], isTrue, reason: '$health'); + final db = await LocalDb.instance; + await db.execute('DROP TABLE observation'); + final broken = await LocalDb.schemaHealth(); + expect(broken['missing_tables'], contains('observation')); + // …and the every-open repair puts it back. + await LocalDb.close(); + await LocalDb.instance; + expect((await LocalDb.schemaHealth())['ok'], isTrue); + }); + // ── the v48 rung, against a REAL export ────────────────────────────────── + // `OPENSTRAP_TEST_DBS=/path/one.db,/path/two.db`. `onUpgrade` runs inside + // openDatabase on iOS's launch-path CPU watchdog, so a rung's cost is a + // number, not an assurance — and phase 3 must move no computed value, which + // means every `day_result` payload has to come back byte-identical. + for (final src in (Platform.environment['OPENSTRAP_TEST_DBS'] ?? '') + .split(',') + .map((s) => s.trim()) + .where((s) => s.isNotEmpty)) { + test( + 'v48 on ${p.basename(src)}: creates the table, moves no number', + () async { + final name = 'v48_${p.basenameWithoutExtension(src)}.db'; + await LocalDb.close(); + LocalDb.dbName = name; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, name)); + final path = p.join(dir, name); + await File(src).copy(path); + + Future> bundlesOf(DatabaseExecutor d) async => { + for (final r in await d.query('day_result')) + '${r['day_id']}|${r['algo_version']}': r['payload_json'], + }; + + // The BEFORE snapshot is read with NO version, so sqflite does not + // migrate the file out from under it. Every later read goes through + // the LocalDb handle: sqflite keeps ONE instance per path, so closing + // a second "read-only" handle closes the live one too. + final plain = await databaseFactory.openDatabase( + path, + options: OpenDatabaseOptions(readOnly: true), + ); + final before = await bundlesOf(plain); + await plain.close(); + expect(before, isNotEmpty); + + final whole = Stopwatch()..start(); + var db = await LocalDb.instance; + whole.stop(); + expect( + await bundlesOf(db), + before, + reason: 'the ladder moved a number', + ); + + // THE v48 RUNG ON ITS OWN. The export is at user_version 27, so the + // open above pays for twenty-one rungs (the v47 re-key dominates). + // Rewinding the stamp on the now-migrated file and re-opening runs + // this rung and nothing else, which is the share it actually costs a + // user upgrading from the shipped build. + await db.execute('DROP TABLE observation'); + await db.execute('PRAGMA user_version = 47'); + await LocalDb.close(); + final rung = Stopwatch()..start(); + db = await LocalDb.instance; + rung.stop(); + // ignore: avoid_print + print( + '[v48] ${p.basename(src)} whole ladder ${whole.elapsedMilliseconds}' + ' ms; v48 rung alone ${rung.elapsedMilliseconds} ms', + ); + expect(await bundlesOf(db), before); + + final info = await db.rawQuery('PRAGMA table_info(observation)'); + expect(info, isNotEmpty); + expect( + (await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type='index' " + "AND tbl_name='observation'", + )).map((r) => r['name']).toSet(), + containsAll(['idx_observation_identity', 'idx_observation_date']), + ); + // The new table starts empty and stays empty: nothing writes it yet. + expect( + ((await db.rawQuery( + 'SELECT COUNT(*) c FROM observation', + )).first['c'] as num).toInt(), + 0, + ); + expect((await LocalDb.schemaHealth())['ok'], isTrue); + await LocalDb.close(); + await File(path).delete(); + }, + timeout: const Timeout(Duration(minutes: 15)), + ); + } + + }); +} diff --git a/test/step_source_ladder_test.dart b/test/step_source_ladder_test.dart index 94bed3da..81020908 100644 --- a/test/step_source_ladder_test.dart +++ b/test/step_source_ladder_test.dart @@ -40,9 +40,16 @@ CoverageSpan _phone(int fromSec, int toSec, int steps) => CoverageSpan( const _h = 3600; /// A 1 Hz substrate carrying [counters] (`-1` = this hardware has no counter). -Substrate _sub(List counters, {int step = 600}) { +/// +/// STAMPED `gen5`, as every real substrate carrying this column is (ingest +/// writes `decoded_onehz.device_family`). Since BANDAGNOSTIC C13 the stamp is +/// what establishes the counter's BEHAVIOUR — cumulative, wraps at 65536, no +/// midnight reset — and an unstamped one abstains rather than sum deltas off a +/// counter that might reset at midnight and lose the day's pre-sync prefix. +Substrate _sub(List counters, {int step = 600, String? family = 'gen5'}) { final n = counters.length; return Substrate( + deviceFamily: family, tsSec: [for (var i = 0; i < n; i++) _t0 + i * step], hr: List.filled(n, 60), rrTsMs: const [], diff --git a/test/substrate_hr_valid_test.dart b/test/substrate_hr_valid_test.dart index 3f99ff7f..4a485221 100644 --- a/test/substrate_hr_valid_test.dart +++ b/test/substrate_hr_valid_test.dart @@ -44,11 +44,16 @@ void main() { expect(s.hrValidAt(1), isNull); }); - test('unknown provenance refuses even when the array carries values', () { - // A pre-schema-41 row, an import or a raw replay has no device stamp. The - // column belongs to gen5, so an unstamped substrate cannot claim it. - final s = _sub(family: null, hrValid: const [1, 1]); - expect(s.hrValidAt(0), isNull); + test('an unstamped substrate that carries real flags is BELIEVED', () { + // BANDAGNOSTIC C12. This used to refuse: `hrValidAt` also required + // `deviceFamily == 'gen5'`, a band id hardcoded in the neutral layer. Only + // a decoder that read the flag off the wire ever writes a non-negative + // value into this column, so a value >= 0 is the source's own declaration — + // and refusing it discarded a measurement because the metadata beside it + // was missing. Absence is still absence; see the gen4 case above. + final s = _sub(family: null, hrValid: const [1, 0]); + expect(s.hrValidAt(0), isTrue); + expect(s.hrValidAt(1), isFalse); }); test('an out-of-range index is absent, not a crash and not false', () { From 501906c450d683e5b44aa9ff479ead794aaa7fbf Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:01:36 +0530 Subject: [PATCH 02/23] ble: a non-whoop device can be seen and connected to now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit two walls made every other band invisible. the scan passed an os-level service filter of the two whoop uuids, so a chest strap advertising 180d never reached our code and the user got "no whoop found". and connect aborted unless four characteristics were present — a generic hrs device has one. that second wall is the whole reason hr_sensor.dart exists as a second parallel ble stack. both now come off a const registry in lib/ble/adapters/. it references protocol's GattProfile and BandProfile rather than restating uuids or header lengths — the only new facts are the three inner-record offsets the engine had inline (opcode, version, counter), and they're required in the constructor so no whoop number hides behind a default. _bandOwner was one process-wide static, so a second engine could never hold a different peripheral. keyed by remoteId now. note this makes two concurrent gatt links possible for the first time, and we don't know the per-oem android cap — logged in the assumptions ledger as needs-device. also: sync_policy's three constants encode whoop's offload model as if it were universal. the time-base one is now countable — a band whose clock is uptime-not-epoch is distinguishable from an unset rtc instead of both just incrementing dropped. the idle timeout and the liveness fuse genuinely need the adapter seam, so they're marked, not guessed at. isGen5 inventory in docs/: 30 sites, 21 data 9 behaviour. two of them say do not unify — the comments record what flipping them broke. --- docs/isgen5-inventory.md | 79 ++++++++++++ lib/ble/adapters/_registry.dart | 124 +++++++++++++++++++ lib/ble/ble_engine.dart | 206 ++++++++++++++++++++------------ lib/ble/ble_state.dart | 22 +++- lib/sync/sync_policy.dart | 61 ++++++++++ test/band_registry_test.dart | 74 ++++++++++++ test/ble_state_test.dart | 26 ++++ 7 files changed, 513 insertions(+), 79 deletions(-) create mode 100644 docs/isgen5-inventory.md create mode 100644 lib/ble/adapters/_registry.dart create mode 100644 test/band_registry_test.dart diff --git a/docs/isgen5-inventory.md b/docs/isgen5-inventory.md new file mode 100644 index 00000000..61d4752e --- /dev/null +++ b/docs/isgen5-inventory.md @@ -0,0 +1,79 @@ +# `isGen5` inventory — DATA vs BEHAVIOUR + +Every band-specific branch in `lib/ble/ble_engine.dart`, classified. This is the +spec for the next wave (change-list D4/D9); **the split itself is not done here**. + +Line numbers are against `ble_engine.dart` at the end of Phase 4 wave 1 +(6,818 lines). Re-derive them before acting — this file rots. + +**DATA** — the branch chooses a *value*: an opcode, a payload, a delay, an +offset, a flag. It belongs in the registry entry / `BandProfile`, and the code +around it becomes unconditional. + +**BEHAVIOUR** — the branch chooses a *different sequence of operations*: an +extra handshake step, a different decoder, a different failure policy. It +belongs in the adapter (`run(BandLink)`). + +**Counts: 30 sites — 21 DATA, 9 BEHAVIOUR.** + +--- + +## DATA (21) + +| Line | Site | The value that differs | Note | +|---|---|---|---| +| 2103 | `_bootstrapAfterRegistration` pre-registration pause | `kGen5PreRegistrationDelay`, gen4 = none | A per-band `Duration`, zero on gen4. The pause code is already shared. | +| 2299 | `_bootstrapAfterRegistration` post-registration pause | `kGen5PostRegistrationDelay` | Same shape as above. | +| 2411 | `_bootstrapSetClock` | whether SET_CLOCK is drift-gated or unconditional | Two-state policy flag. The gate (`BootstrapClockGate`) is already pure and shared; only "does this band use it" differs. **gen4's unconditional write is deliberate** — the comment says the gen5 evidence does not transfer. Carry the flag, do not unify the behaviour. | +| 2549–2557 | keep-alive live re-arm | which command re-arms live: IMU toggle vs `sendR10R11Realtime 0x01` | A per-band "live re-arm command set". | +| 3204 | `_offloadPayload` | `[0x00]` vs `[]` | Pure payload. Already centralised — the cleanest DATA site in the file. | +| 3436 | console-log line | whether this band's console output is logged | Debug-visibility only. NOTE the gate is not describing the wire: protocol's `decodeFrame` produces `console_log` for `PacketType.consoleLogs` on BOTH bands (`control.dart:1352`), so a gen4 strap emitting one is silently suppressed here. A per-band "log the console" bool, or just delete the gate. | +| 3643 | `kKnownRecordVersions` membership | the record versions this band's decoder claims | Per-band set, today a gen4-only global. See BEHAVIOUR/3603 — the two move together. | +| 4578 | `gateEnforced = session.band.isGen5` | whether `expectedPacketCount` is trustworthy enough to gate a burst | One bool. **Do not flip gen4's** — an enforced gate on gen4 stalls the drain permanently (comment at 4572). | +| 5321, 5332 | `setClock` | *nothing but the log string* | The body is byte-identical on both. This branch can be deleted outright and the label read from `BandEntry.label`. | +| 5724, 5732, 5736 | `setAlarm` payload + log | `AlarmPayloads.setPayloadForBand(isGen5:)` | Already delegated to a pure policy in `ble_state.dart`; the policy takes `isGen5:` and would take the entry (or a payload-shape field) instead. Sibling-owned file — coordinate. | +| 5787 | `getAlarm` payload | `AlarmPayloads.getPayloadForBand` | Same as above. | +| 5801 | `runAlarm` | opcode + payload (`runHapticPatternMaverick` vs `runAlarm`) | Opcode swap. The "do NOT stop haptics first" note is a comment, not a code difference. | +| 5820 | `disableAlarm` payload | `AlarmPayloads.disableForBand` | Same as 5724. | +| 5828 | `getStrapName` | opcode + payload | Opcode swap. | +| 5844 | `setStrapName` | opcode | Body identical on both — the comment says so. | +| 5853 | `getHello` | opcode + payload (`0x91`/`[0x01]` vs Harvard/`[0x00]`) | Opcode swap. | +| 5864 | `buzzPattern` | opcode + payload; gen5 ignores `pattern` | Opcode swap plus "this band has one fixed waveform" — a payload fact. | +| 5894, 5907, 5912 | `enableLiveStreams` | which toggles are in the ON set | A per-band command LIST, not a branch. **Carries a real footgun**: `enableOpticalData` is the SAVE-to-history toggle on gen5 (comment at 5900). The list is the safety boundary. | +| 5951 | `enableHrOnlyLive` | which toggles are in the OFF set | Same list shape. | +| 5967 | `disableLiveStreams` | which toggles are in the OFF set | Same list shape. Three sites, one per-band list. | +| 6170, 6187 | `_maybeAugmentClockEpoch` | GET_CLOCK opcode; body offset `3` vs `2` | Offset + opcode. Exactly the D3 shape already moved into `BandEntry` for the historical-record offsets. | + +## BEHAVIOUR (9) + +| Line | Site | What actually differs | +|---|---|---| +| 2328 | `_readGen5Hello()` inside `_bootstrapAfterRegistration` | An extra handshake round-trip with its own link-death abort, ordered before the clock read. gen4 has no equivalent step. | +| 2434 | `_readAdvertisingNameGen5` | A gen5-only setup command with its own "not a readiness gate" failure semantics. | +| 2461 | `_maybeStartBatteryPackFollowUp` | A gen5-only background task with its own retry schedule and once-per-session latch. | +| 3604–3643 | historical record dispatch | `decodeGen5HistoricalSample` vs the gen4 version-routed chain, and which record kinds fall through to `raw_archive`. The core adapter responsibility. | +| 5102 | `enableGen5DeepBuffers` | A gen5-only multi-frame `SET_FF_VALUE` sequence behind the **one audited `allowDangerous: true`**. Whatever holds it must keep the dangerous-opcode block's carve-out explicit and single. | +| 5128–5181 | `sendInit` | The whole handshake: gen5 = CLIENT_HELLO already done + `GET_DATA_RANGE` + `SEND_HISTORICAL_DATA`; gen4 = the 5-packet INIT loop. Two different state machines sharing one method. **Biggest single item.** | +| 5701 | `setAlarm` pre-arm | gen5 issues a SET_CLOCK + 120 ms settle before arming; gen4 does not. A sequence, not a payload. | +| 5146–5161 | gen5 deep-buffer ordering inside `sendInit` | Config flags must land *before* the offload trigger. An ordering constraint that only exists on one band. | +| 6396 | `DrainController.onArchiveHistorical` reads `inner[1]` | Left as-is deliberately: per MULTIBAND_PLAN §3.1 `DrainController` stays in `ble_engine` as the **gen4 adapter's private collaborator**. Its WHOOP assumptions are correct where they are; it should be named honestly, not made band-generic. | + +--- + +## What this inventory says about the next wave + +1. **21 of 30 are opcode/payload/offset/delay values.** Most of the `isGen5` + surface is not behaviour at all — it is a table. A `BandEntry` that carries + an opcode map, three command lists (live-on / live-off / re-arm), two + durations and three bools erases two thirds of the branches without an + adapter existing. +2. **Three of the nine BEHAVIOUR sites are one thing**: the gen5 setup + sequence (2328, 2434, 2461). They are ordered steps of one handshake that + gen4 does not have. +3. **The alarm payload sites already have the right shape** — a pure policy in + `ble_state.dart` taking a band discriminator. Widening `isGen5:` to the + entry is mechanical; that file is owned elsewhere, so it needs coordinating, + not redesigning. +4. **Two branches must NOT be unified** even though they look like duplication: + the gen4 unconditional SET_CLOCK (2411) and the gen4 advisory-only burst + count gate (4578). Both comments record a failure that flipping them caused. diff --git a/lib/ble/adapters/_registry.dart b/lib/ble/adapters/_registry.dart new file mode 100644 index 00000000..ef7fdf20 --- /dev/null +++ b/lib/ble/adapters/_registry.dart @@ -0,0 +1,124 @@ +// The const table of bands this build can see. +// +// Dart AOT has no runtime code loading, no `dart:mirrors`, and lazy static +// initialisation defeats import-for-side-effects registration (ASSUMPTIONS +// E4), so this is a hand-maintained `const` list. Adding a band is a source +// edit here and nothing else. Revisit past ~50 entries. +// +// SCOPE — this is the MINIMUM registry, not the adapter seam. It holds only +// the facts `ble_engine.dart` used to hardcode: +// +// • which service UUIDs the scan filters on (D1) +// • which characteristics a link must expose to connect (D2) +// • where the inner-record fields sit (D3) +// +// There is deliberately NO `run(BandLink)`, no `BandEvent`, no `InputSignal` +// and — per MULTIBAND_PLAN §3.1 — no capability booleans, ever. Declare the +// INPUT signals a device physically emits when that lands (D9); never a +// `supportsX()` claim about our own features. +// +// ponytail: every entry here is a framed WHOOP-family band, because +// [BandProfile]/[GattProfile] cannot yet express anything else — a u8 length, +// no CRC, or three characteristics (change-list D7). This file is the place +// that stops being true in; nothing above it needs to change again. + +import 'package:openstrap_protocol/openstrap_protocol.dart'; + +/// One band the app can discover and connect to. +/// +/// The wire format itself stays in `protocol` ([BandProfile] = header length, +/// size-field offset, direction markers; [GattProfile] = the UUID map). This +/// type carries the edge-side facts that live above the codec — discovery and +/// the inner-record field offsets — following the same "it is data, not a +/// branch" pattern rather than inventing a parallel one. +class BandEntry { + /// Stable identifier. Stamped into `DeviceState.generation` and, downstream, + /// `device_family` — so it is a storage key: never rename a shipped one. + final String id; + + /// Human label for logs and (later) the pairing UI. + final String label; + + /// GATT UUID map for this band, straight from `protocol`. + final GattProfile gatt; + + /// Frame envelope profile — header length, size-field offset, header CRC. + final BandProfile wire; + + /// The characteristics a link MUST expose or the connect aborts. + /// + /// Defaults to this entry's own four command/notify characteristics, which + /// is what a WHOOP link genuinely needs. It is a FIELD and not a constant + /// because demanding four unconditionally is why a second, parallel BLE + /// stack (`hr_sensor.dart`) had to exist at all: a generic HRS device + /// exposes one notify characteristic and nothing else. + final List? _requiredCharacteristics; + + /// Offset of the opcode byte within the inner payload + /// (`[pktType, seq, opcode, body…]`). + final int innerOpcodeOffset; + + /// Offset of the record-version byte within a historical record's inner + /// payload. + final int innerVersionOffset; + + /// Offset of the u32-LE record counter within a historical record's inner + /// payload. + final int innerCounterOffset; + + const BandEntry({ + required this.id, + required this.label, + required this.gatt, + required this.wire, + required this.innerOpcodeOffset, + required this.innerVersionOffset, + required this.innerCounterOffset, + List? requiredCharacteristics, + }) : _requiredCharacteristics = requiredCharacteristics; + + /// Service UUID to advertise-filter the scan on. + String get service => gatt.service; + + /// 32-bit prefix used to match this band's service from a scan result or a + /// discovered service list (case-insensitive `startsWith`). + String get servicePrefix => gatt.servicePrefix; + + List get requiredCharacteristics => + _requiredCharacteristics ?? + [gatt.cmdTo, gatt.cmdFrom, gatt.events, gatt.data]; + + /// Index of the opcode byte in a fully-framed packet. + int get frameOpcodeIndex => wire.headerLen + innerOpcodeOffset; +} + +/// WHOOP 4 ("Harvard", 6108xxxx). +const BandEntry kWhoopGen4 = BandEntry( + id: 'gen4', + label: 'WHOOP 4', + gatt: GattProfile.gen4, + wire: BandProfile.gen4, + innerOpcodeOffset: 2, + innerVersionOffset: 1, + innerCounterOffset: 3, +); + +/// WHOOP 5 / MG ("fd4b"). Same inner payload layout as gen4 — only the +/// envelope differs, which is exactly what [BandProfile] models. +const BandEntry kWhoopGen5 = BandEntry( + id: 'gen5', + label: 'WHOOP 5', + gatt: GattProfile.gen5, + wire: BandProfile.gen5, + innerOpcodeOffset: 2, + innerVersionOffset: 1, + innerCounterOffset: 3, +); + +/// Every band this build can see. Order is match order during discovery. +const List kBandRegistry = [kWhoopGen4, kWhoopGen5]; + +/// The entry speaking [wire]. Used by the engine's test seam, which is handed +/// a [BandProfile] rather than an entry. +BandEntry bandEntryFor(BandProfile wire) => + kBandRegistry.firstWhere((e) => e.wire.type == wire.type); diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index 9d56505f..07d39e58 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -47,6 +47,7 @@ import '../data/models.dart'; import '../platform/tasker_bridge.dart'; import '../sync/paired_device.dart' show cleanDeviceLabel; import '../sync/sync_policy.dart'; +import 'adapters/_registry.dart'; import 'ble_state.dart'; // Little-endian u32 reader. The package keeps `u32` private, and the engine only @@ -457,11 +458,15 @@ class _Session { final BluetoothDevice device; BluetoothCharacteristic? cmdTo; - /// Which WHOOP generation this link speaks. Defaults to gen4 (WHOOP 4) and is + /// Which registered band this link speaks. Defaults to gen4 (WHOOP 4) and is /// pinned once during service discovery via [applyBand] — everything that - /// differs by generation (frame header/CRC, GATT UUIDs, command envelope, - /// history ACK, record decode) reads from here. - BandProfile band = BandProfile.gen4; + /// differs by band (frame header/CRC, GATT UUIDs, command envelope, history + /// ACK, record field offsets) reads from here. + BandEntry entry = kWhoopGen4; + + /// This link's frame envelope profile. Owned by [entry]; kept as a getter + /// because the wire format is `protocol`'s to define, not edge's. + BandProfile get band => entry.wire; final Map asm = { 'cmd_from': FrameReassembler(), @@ -469,13 +474,13 @@ class _Session { 'data': FrameReassembler(), }; - /// Pin this session's generation and rebuild the reassemblers with the - /// matching header shape. Called once, at discovery, before any frame is fed. - void applyBand(BandProfile b) { - band = b; - asm['cmd_from'] = FrameReassembler(profile: b); - asm['events'] = FrameReassembler(profile: b); - asm['data'] = FrameReassembler(profile: b); + /// Pin this session's band and rebuild the reassemblers with the matching + /// header shape. Called once, at discovery, before any frame is fed. + void applyBand(BandEntry e) { + entry = e; + asm['cmd_from'] = FrameReassembler(profile: e.wire); + asm['events'] = FrameReassembler(profile: e.wire); + asm['data'] = FrameReassembler(profile: e.wire); } final List subs = []; Timer? heartbeat; @@ -665,12 +670,23 @@ class BleEngine { // stuck at 28, duplicate ACKs, sync never completes). Enforce a single owner, // FOREGROUND-PRIORITY: a background drainer yields if the band is already owned; // a foreground engine preempts a background owner by dropping its link. - static BleEngine? _bandOwner; - - /// Claim exclusive ownership of the band for this engine. Returns false only for - /// a background drainer when another engine already owns it (→ it must NOT touch - /// the band this cycle). A foreground engine always succeeds and preempts any - /// background owner by disconnecting it. + // + // KEYED BY `remoteId`, because the hazard above is about ONE peripheral: two + // engines connected to two DIFFERENT devices never share a trim cursor, and + // a process-wide single owner made a second device's connect preempt the + // primary's link on every connect. + static final Map _bandOwners = {}; + + /// The peripheral this engine currently holds a claim on, if any. Kept + /// because [_releaseBand] is called from paths with no device in scope, and + /// it must release exactly the key it took. + String? _claimedBandId; + + /// Claim exclusive ownership of the peripheral [remoteId] for this engine. + /// Returns false only for a background drainer when another engine already + /// owns THAT peripheral (→ it must NOT touch it this cycle); a claim on a + /// different device does not block it. A foreground engine always succeeds + /// and preempts any background owner of the same device by disconnecting it. /// /// SERIALIZED PREEMPTION: the preempted engine's teardown is AWAITED (bounded) /// before we proceed — firing our connect while its disconnect is still in @@ -679,8 +695,11 @@ class BleEngine { /// GATT). A hung teardown can't wedge us forever: after the timeout we log and /// proceed (the preempted engine's own session guards make its late teardown /// harmless once we own the band). - Future _claimBand() async { - final other = _bandOwner; + Future _claimBand(String remoteId) async { + // Moving to a different peripheral: let the old one go first, or this + // engine holds two keys and the stale one starves a later drain. + if (_claimedBandId != null && _claimedBandId != remoteId) _releaseBand(); + final other = _bandOwners[remoteId]; final incumbentPresent = other != null && !identical(other, this); final decision = BandClaimPolicy.decide( incumbentPresent: incumbentPresent, @@ -716,7 +735,8 @@ class BleEngine { } break; } - _bandOwner = this; + _bandOwners[remoteId] = this; + _claimedBandId = remoteId; return true; } @@ -730,17 +750,20 @@ class BleEngine { _phase != BleConnState.idle && _phase != BleConnState.error; - /// Test-only view of the process-wide single-owner claim. + /// Test-only view of the per-peripheral single-owner claim. @visibleForTesting - static bool get bandClaimed => _bandOwner != null; + static bool get bandClaimed => _bandOwners.isNotEmpty; - /// Test-only reset of the process-wide claim (static state otherwise leaks - /// across test cases). + /// Test-only reset of the claims (static state otherwise leaks across test + /// cases). @visibleForTesting - static void resetBandClaimForTest() => _bandOwner = null; + static void resetBandClaimForTest() => _bandOwners.clear(); void _releaseBand() { - if (identical(_bandOwner, this)) _bandOwner = null; + final id = _claimedBandId; + if (id == null) return; + if (identical(_bandOwners[id], this)) _bandOwners.remove(id); + _claimedBandId = null; } // ── transport state machine ───────────────────────────────────────────────── @@ -1027,7 +1050,7 @@ class BleEngine { ); session.connected = true; session.sawConnected = true; - session.applyBand(band); + session.applyBand(bandEntryFor(band)); _session = session; debugWriteHook = onWrite; _drain = DrainController( @@ -1723,7 +1746,7 @@ class BleEngine { /// Serialised process-wide through [withScanLock]: the HR-sensor scan shares /// this one radio scanner, and the `isScanning == false` await below is /// satisfied by ITS `stopScan` too — an unserialised scan silently ends - /// having seen nothing and reports "No WHOOP found". + /// having seen nothing and reports "No band found". Future scan({ Duration timeout = const Duration(seconds: 12), }) => @@ -1745,10 +1768,12 @@ class BleEngine { await FlutterBluePlus.stopScan(); } _setPhase(BleConnState.scanning); - // Advertise-filter on BOTH generations' service UUIDs (gen4 6108xxxx + - // gen5 fd4bxxxx); the actual generation is pinned later at discovery. - final gen4Svc = Guid(GattProfile.gen4.service); - final gen5Svc = Guid(GattProfile.gen5.service); + // Advertise-filter on every registered band's service UUID. This is an + // OS-LEVEL filter: a device whose service is not in this list is invisible + // to the callback below, so the registry — not a literal here — is what + // decides which bands can be seen at all. The actual band is pinned later + // at discovery. + final wanted = [for (final e in kBandRegistry) Guid(e.service)]; BluetoothDevice? found; final sub = FlutterBluePlus.onScanResults.listen((results) { for (final r in results) { @@ -1756,18 +1781,20 @@ class BleEngine { final advNames = r.advertisementData.serviceUuids.map( (g) => g.str.toLowerCase(), ); + // ponytail: `whoop` name-match is a WHOOP-only fallback for a band that + // advertises its name but not its service UUID. A per-entry name + // matcher is D9's `BandDiscovery`; until then this one literal stays. if (found == null && (name.contains('whoop') || advNames.any((s) => - s.startsWith('61080001') || s.startsWith('fd4b0001')))) { + kBandRegistry.any((e) => s.startsWith(e.servicePrefix))))) { found = r.device; FlutterBluePlus.stopScan(); } } }); try { - await FlutterBluePlus.startScan( - withServices: [gen4Svc, gen5Svc], timeout: timeout); + await FlutterBluePlus.startScan(withServices: wanted, timeout: timeout); await FlutterBluePlus.isScanning.where((on) => on == false).first; } catch (e) { // Android reports a missing runtime permission by throwing here rather @@ -1785,7 +1812,10 @@ class BleEngine { } if (found == null) { _setPhase(BleConnState.idle); - _log('No WHOOP found (force-quit the official app; band must be free).'); + // The remedy in this line is still WHOOP-specific ("the official app"). + // Per-band copy needs the per-entry discovery/label of D9; the registry + // does not make it fixable on its own. + _log('No band found (force-quit the official app; band must be free).'); } else { _clearBlocker(); } @@ -1866,7 +1896,7 @@ class BleEngine { // band the foreground session already owns (duplicate ACKs corrupt the trim // cursor). Foreground engines preempt instead — awaiting the preempted // engine's teardown so two FBP ops never overlap. See [_claimBand]. - if (!await _claimBand()) return false; + if (!await _claimBand(device.remoteId.str)) return false; // Any prior session is dead to us now — tear it down before a new one. await _teardownSession(intentional: true); try { @@ -1889,7 +1919,7 @@ class BleEngine { /// down, drop to `idle`, AND RELEASE THE BAND CLAIM. /// /// [_claimBand] runs BEFORE the link is up, so a connect that threw used to - /// leave `_bandOwner` pointing at an engine with no link — and only + /// leave the claim pointing at an engine with no link — and only /// `disconnect()` ever released it, which nothing calls on this path. Every /// later background drain then saw a non-null owner and yielded forever. Future _failConnect() async { @@ -2013,52 +2043,59 @@ class BleEngine { final services = await device .discoverServices() .timeout(_serviceDiscoveryTimeout); - // Pin the generation from whichever service the peripheral exposes: - // gen4 "Harvard" 6108xxxx, or gen5 "fd4b" fd4bxxxx. This drives the frame - // header/CRC, command envelope, ACK, and record decode for the session. + // Pin the band from whichever registered service the peripheral exposes. + // This drives the frame header/CRC, command envelope, ACK, and record + // decode for the session. BluetoothService? svc; - BandProfile band = BandProfile.gen4; + BandEntry? entry; for (final s in services) { final u = s.uuid.str.toLowerCase(); - if (u.startsWith(GattProfile.gen4.servicePrefix)) { - svc = s; - band = BandProfile.gen4; - break; - } - if (u.startsWith(GattProfile.gen5.servicePrefix)) { - svc = s; - band = BandProfile.gen5; - break; + for (final e in kBandRegistry) { + if (u.startsWith(e.servicePrefix)) { + svc = s; + entry = e; + break; + } } + if (svc != null) break; } - if (svc == null) { - _log('No WHOOP service (gen4 6108xxxx / gen5 fd4bxxxx) found on device.'); + if (svc == null || entry == null) { + _log('No known band service found on device (looked for: ' + '${kBandRegistry.map((e) => "${e.servicePrefix}xxxx").join(", ")}).'); await _failConnect(); return false; } - session.applyBand(band); - state.generation = band.isGen5 ? 'gen5' : 'gen4'; - _log('Detected ${band.isGen5 ? "WHOOP 5 (gen5)" : "WHOOP 4 (gen4)"} link.'); - final gatt = band.gatt; - BluetoothCharacteristic? find(String prefix) { + session.applyBand(entry); + state.generation = entry.id; + _log('Detected ${entry.label} (${entry.id}) link.'); + final band = entry.wire; + final gatt = entry.gatt; + BluetoothCharacteristic? find(String uuid) { + final prefix = uuid.substring(0, 8); for (final c in svc!.characteristics) { if (c.uuid.str.toLowerCase().startsWith(prefix)) return c; } return null; } - session.cmdTo = find(gatt.cmdTo.substring(0, 8)); - final cmdFrom = find(gatt.cmdFrom.substring(0, 8)); - final events = find(gatt.events.substring(0, 8)); - final data = find(gatt.data.substring(0, 8)); - if (session.cmdTo == null || - cmdFrom == null || - events == null || - data == null) { - _log('Missing one or more ${band.isGen5 ? "fd4b" : "Harvard"} characteristics.'); + // WHICH characteristics a link must expose is registry data. Demanding + // all four unconditionally is why `hr_sensor.dart` exists as a second + // parallel BLE stack — a generic HRS device has ONE notify + // characteristic and would abort here. + final missing = [ + for (final u in entry.requiredCharacteristics) + if (find(u) == null) u.substring(0, 8), + ]; + if (missing.isNotEmpty) { + _log('${entry.label}: missing required characteristic(s) ' + '${missing.join(", ")}.'); await _failConnect(); return false; } + session.cmdTo = find(gatt.cmdTo); + final cmdFrom = find(gatt.cmdFrom); + final events = find(gatt.events); + final data = find(gatt.data); // (gen5 only — see [kGen5PreRegistrationDelay]): // the bond is complete by here, so this is the 600 ms that precedes @@ -2072,9 +2109,11 @@ class BleEngine { return false; } _setPhase(BleConnState.subscribing); - await _subscribe(session, cmdFrom, 'cmd_from'); - await _subscribe(session, events, 'events'); - await _subscribe(session, data, 'data'); + // Null only for a band whose entry does not require the characteristic — + // the `missing` gate above has already aborted for one that does. + if (cmdFrom != null) await _subscribe(session, cmdFrom, 'cmd_from'); + if (events != null) await _subscribe(session, events, 'events'); + if (data != null) await _subscribe(session, data, 'data'); if (!await _bootstrapAfterRegistration(session)) return false; // Fresh clock verification stamp — see kRtcReverifyIntervalSeconds. @@ -2995,7 +3034,7 @@ class BleEngine { // FORCE_TRIM (whose full-erase form is two 0xFEFEFEFE args), REBOOT and // POWER_CYCLE cannot leave this engine by any path. final opcode = - allowDangerous ? null : _opcodeOfFrame(raw, session?.band ?? BandProfile.gen4); + allowDangerous ? null : _opcodeOfFrame(raw, session?.entry ?? kWhoopGen4); if (opcode != null && (dangerousCmds.contains(opcode) || OpcodeSafety.isDestructive(opcode))) { _log('REFUSED dangerous opcode 0x${opcode.toRadixString(16)} at _write'); @@ -3048,10 +3087,15 @@ class BleEngine { } /// The command opcode carried by an already-framed outbound write, or null - /// when [raw] is too short to carry one. Layout is `header | pktType | seq | - /// opcode | body…`, and only the header length differs by generation. - static int? _opcodeOfFrame(Uint8List raw, BandProfile band) { - final i = band.headerLen + 2; + /// when [raw] is too short to carry one. Where the opcode sits is registry + /// data ([BandEntry.frameOpcodeIndex] = the band's header length plus its + /// inner-payload opcode offset), not a WHOOP literal. + /// + /// This feeds the dangerous-opcode block, so a band whose entry gets this + /// wrong reads the wrong byte and the block stops protecting it — see the + /// guard's own test. + static int? _opcodeOfFrame(Uint8List raw, BandEntry entry) { + final i = entry.frameOpcodeIndex; return i < raw.length ? raw[i] : null; } @@ -3536,7 +3580,9 @@ class BleEngine { void _ingestHistoricalFrame(Frame frame) { final pt = frame.packetType; if (pt != PacketType.historicalData) return; - final recType = frame.inner.length > 1 ? frame.inner[1] : -1; + // Where the record-version byte sits is registry data, not a literal. + final vAt = (_session?.entry ?? kWhoopGen4).innerVersionOffset; + final recType = frame.inner.length > vAt ? frame.inner[vAt] : -1; final counter = _counterFromInner(frame.inner); // Explicit, observable band-reboot signal — see CounterRegressionDetector. // 0 is _counterFromInner's fallback for a too-short frame, not a real @@ -5017,8 +5063,12 @@ class BleEngine { } } - int _counterFromInner(Uint8List inner) => - inner.length >= 7 ? u32(inner, 3) : 0; + /// The record counter out of a historical record's inner payload. Where it + /// sits is registry data; 0 is the too-short fallback, not a real counter. + int _counterFromInner(Uint8List inner) { + final at = (_session?.entry ?? kWhoopGen4).innerCounterOffset; + return inner.length >= at + 4 ? u32(inner, at) : 0; + } static const _hexDigits = '0123456789abcdef'; // Called once per stored record, once per archived record and once per live // frame, so it runs ~50k times in an offload on the UI isolate. The obvious diff --git a/lib/ble/ble_state.dart b/lib/ble/ble_state.dart index e9566311..5a61ebc5 100644 --- a/lib/ble/ble_state.dart +++ b/lib/ble/ble_state.dart @@ -15,7 +15,7 @@ import 'dart:math'; import 'package:openstrap_protocol/openstrap_protocol.dart' show alarmRev1Payload; -import '../sync/sync_policy.dart' show isPlausibleUnix; +import '../sync/sync_policy.dart' show isPlausibleUnix, kMinPlausibleUnix; /// The explicit connection state machine. The flutter_blue_plus connection-state /// stream is the SOURCE OF TRUTH for connected/disconnected; this enum layers the @@ -487,8 +487,27 @@ class RecordGate { /// Records rejected by the plausibility gate this connection. int dropped = 0; + /// Of [dropped], the ones that failed the ABSOLUTE floor + /// (`ts < kMinPlausibleUnix`) rather than the future or session-window tests. + /// The two have opposite prognoses: an unset or wandering RTC drifts back + /// into range (or SET_CLOCK pulls it back), while a source whose time base is + /// not a wall-clock epoch at all never will. Counted, never acted on — the + /// gate's verdict is identical either way, and must stay so. + int droppedBelowFloor = 0; + RecordGate({this.frontierTs = 0}); + /// True when this connection rejected records and EVERY rejection was below + /// the absolute floor — the signature of a source that does not stamp + /// wall-clock time (uptime-since-boot, a sequence number, milliseconds). + /// + /// Worth reporting apart from a transient clock problem because the stall is + /// PERMANENT: no retry, reconnect or SET_CLOCK resolves it, and the visible + /// symptom (records seen, nothing banked, the chunk re-delivered forever) is + /// identical to the transient case. See the `kMinPlausibleUnix` assumption + /// block in `sync_policy.dart` for the upgrade path. + bool get timeBaseNotWallClock => dropped > 0 && dropped == droppedBelowFloor; + /// Should this record be stored? Records with no decodable time ([tsEpoch] /// null or <= 0) are always admitted (we can't gate them) and never advance /// the frontier. Plausible records advance [frontierTs]; implausible ones @@ -507,6 +526,7 @@ class RecordGate { sessionNewestUnix: sessionNewestUnix, )) { dropped++; + if (tsEpoch < kMinPlausibleUnix) droppedBelowFloor++; return false; } if (tsEpoch > frontierTs) frontierTs = tsEpoch; diff --git a/lib/sync/sync_policy.dart b/lib/sync/sync_policy.dart index 9a49c8c5..00aaccfc 100644 --- a/lib/sync/sync_policy.dart +++ b/lib/sync/sync_policy.dart @@ -16,7 +16,52 @@ import 'dart:math' as math; const int kBackfillIntervalSeconds = 900; // re-offload every 15 min (periodic) const int kKeepAliveIntervalSeconds = 30; // re-arm realtime, poll battery, watchdog +/// How long an open offload burst may sit silent before its un-committed chunk +/// is abandoned (`BleEngine._armIdleWatchdog` → `DrainController.discardOpenChunk`). +/// +/// ASSUMES: a healthy transfer never pauses this long, and abandoning the open +/// chunk costs nothing because the band re-delivers it on the next offload. +/// Both halves are WHOOP's flash-and-trim model — a draining band emits records +/// back-to-back, and un-ACKed flash is kept by contract. +/// FALSIFIED BY: any source whose NORMAL transfer pauses longer — a +/// fetch-by-range device answering one page per request, a rate-limited +/// transport, a band that idles between stored sessions inside one transfer. +/// WHEN WRONG: every attempt discards its own work and restarts, so the device +/// makes ZERO forward progress, permanently. The per-session retry cap +/// (`_maxHistoricalRetriesPerSession`) only hands the same loop to the +/// reconnect path; nothing counts it across sessions and nothing tells the +/// user. It is at least LOUD — `[SYNC] idle watchdog` is logged on every +/// expiry — so it is a stall findable in a log, not a silent one. +/// HOW TO CHECK: two `[SYNC] idle watchdog` lines with no durable commit +/// between them, repeating across reconnects. +// ponytail: one timeout for every source, sized for WHOOP's continuous drain. +// Upgrade path = the adapter declares its own maximum healthy inter-frame pause +// AND whether an abandoned chunk is re-delivered at all. The cross-session +// escalation to hang the "this keeps happening" report on already exists +// ([NoDurableProgressEscalation]); the discard path just does not feed it. +// Until both exist, do not point a slower source at this drain. const int kBackfillIdleTimeoutSeconds = 60; // strap went silent mid-offload + +/// Silence past which an ACTIVE session tears itself down (`_keepAliveFire`). +/// +/// ASSUMES: a healthy link always produces inbound traffic inside two minutes. +/// On WHOOP that holds only because WE manufacture the traffic — the 30 s +/// keep-alive forces a GET_BATTERY_LEVEL once silence passes +/// [kNoStreamPollSilenceSeconds] (or half this fuse, with a live stream armed). +/// So the fuse measures "did our own poll come back", not "is this band alive". +/// FALSIFIED BY: a source with no cheap pollable characteristic, or one that +/// legitimately says nothing between readings — a fetch-by-range band, a sensor +/// that notifies only on a detected beat. +/// WHEN WRONG: the link is bounced every two minutes forever, each bounce +/// costing a reconnect and re-handshake, so the device may never stay up long +/// enough to transfer anything. +/// HOW TO CHECK: `No data for >120s — bouncing the link.` repeating on a ~2 min +/// cadence with no error between the bounces. +// ponytail: the fuse and the keep-alive poll that satisfies it are ONE +// mechanism, and both are WHOOP-shaped. Upgrade path = the adapter declares the +// interval at which a healthy link of its kind produces inbound traffic (none = +// no fuse) and what to poll to provoke it. [isLinkStale] already has the right +// shape — one bar per traffic mode — it just cannot know a second band's modes. const int kLivenessFuseSeconds = 120; // no data for >fuse ⇒ bounce the link // Every existing RTC recheck is symptom-triggered (drift detected on the ONE // GET_CLOCK read at connect, or a defensive SET_CLOCK on StuckStrapDetector @@ -70,6 +115,22 @@ bool isLinkStale(Duration sinceLastRx, {bool liveStreamArmed = true}) => (liveStreamArmed ? kLinkFreshnessSeconds : kLinkFreshnessNoStreamSeconds); // ── plausibility gates (unix seconds) ──────────────────────────────────────── +/// ASSUMES: every source stamps records with an ABSOLUTE wall-clock epoch, so a +/// value under the 2023-11 floor is junk — an unset RTC, a previous owner's +/// garbage, a misread offset. True of WHOOP, which carries a settable RTC. +/// FALSIFIED BY: a source whose time base is uptime-since-boot, a sequence +/// number, or milliseconds — each lands permanently under the floor. +/// WHEN WRONG: every record is refused. Nothing is LOST (`TrimAckPolicy` +/// correctly refuses the trim on a drop-only burst, so the band keeps its +/// flash) but nothing is stored either, and the same chunk is re-delivered +/// forever. [RecordGate.timeBaseNotWallClock] is what makes that case tellable +/// from a transient wandering clock, which looks identical at the gate. +/// HOW TO CHECK: `gate_dropped_total` climbing while `records_seen` stays flat, +/// with `RecordGate.droppedBelowFloor == RecordGate.dropped`. +// ponytail: an absolute floor, so a relative time base cannot be rescued here +// at all. Upgrade path = the adapter supplies an epoch ANCHOR (the wall time +// its zero corresponds to) and converts BEFORE the gate; the gate itself stays +// absolute, which is the only reason it can still reject a wandering RTC. const int kMinPlausibleUnix = 1700000000; // 2023-11 floor const int kFutureMargin = 86400; // +1 day const int kSessionRangeMargin = diff --git a/test/band_registry_test.dart b/test/band_registry_test.dart new file mode 100644 index 00000000..bc73aac1 --- /dev/null +++ b/test/band_registry_test.dart @@ -0,0 +1,74 @@ +// The band registry is the ONE place the BLE engine's device-specific facts +// live (change-list D1/D2/D3). Every value here used to be a literal in +// `ble_engine.dart`, so this pins them at exactly what the engine did before — +// a wrong offset does not throw, it silently reads the wrong byte. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/adapters/_registry.dart'; +import 'package:openstrap_protocol/openstrap_protocol.dart'; + +void main() { + test('ids are unique and stable — they are stamped as device_family', () { + expect(kBandRegistry.map((e) => e.id).toList(), ['gen4', 'gen5']); + }); + + test('D1 — the scan service list is exactly the two WHOOP services', () { + expect(kBandRegistry.map((e) => e.service).toList(), [ + GattProfile.gen4.service, + GattProfile.gen5.service, + ]); + expect(kBandRegistry.map((e) => e.servicePrefix).toList(), + ['61080001', 'fd4b0001']); + }); + + test('D2 — a WHOOP link requires its four command/notify characteristics', + () { + for (final e in kBandRegistry) { + expect(e.requiredCharacteristics, [ + e.gatt.cmdTo, + e.gatt.cmdFrom, + e.gatt.events, + e.gatt.data, + ]); + } + }); + + test('D2 — an entry may require fewer (a generic HRS has one notify char)', + () { + const one = BandEntry( + id: 'x', + label: 'x', + gatt: GattProfile.gen4, + wire: BandProfile.gen4, + innerOpcodeOffset: 2, + innerVersionOffset: 1, + innerCounterOffset: 3, + requiredCharacteristics: ['61080005-x'], + ); + expect(one.requiredCharacteristics, hasLength(1)); + }); + + test('D3 — frameOpcodeIndex lands on the opcode of a real built frame', () { + // This is the byte the dangerous-opcode block reads. If it moves, the + // block silently stops blocking. + for (final e in kBandRegistry) { + final raw = buildCommand(7, Cmd.rebootStrap, const [0x01], e.wire); + expect(raw[e.frameOpcodeIndex], Cmd.rebootStrap, reason: e.id); + } + expect(kWhoopGen4.frameOpcodeIndex, 6); // 4-byte header + 2 + expect(kWhoopGen5.frameOpcodeIndex, 10); // 8-byte header + 2 + }); + + test('D3 — historical inner offsets are unchanged from the old literals', + () { + for (final e in kBandRegistry) { + expect(e.innerVersionOffset, 1, reason: e.id); // inner[1] + expect(e.innerCounterOffset, 3, reason: e.id); // u32(inner, 3) + } + }); + + test('bandEntryFor maps a wire profile back to its entry', () { + expect(bandEntryFor(BandProfile.gen4).id, 'gen4'); + expect(bandEntryFor(BandProfile.gen5).id, 'gen5'); + }); +} diff --git a/test/ble_state_test.dart b/test/ble_state_test.dart index d103e5b2..d4cd2576 100644 --- a/test/ble_state_test.dart +++ b/test/ble_state_test.dart @@ -218,6 +218,32 @@ void main() { expect(g.dropped, 1); }); + test('a below-floor time base is tellable from a wandering clock', () { + // Uptime-since-boot: every stamp is under kMinPlausibleUnix, forever. + final uptime = RecordGate(); + expect(uptime.admit(3600, wallNow: wall), isFalse); + expect(uptime.admit(7200, wallNow: wall), isFalse); + expect(uptime.droppedBelowFloor, 2); + expect(uptime.timeBaseNotWallClock, isTrue); + + // A wandering/future RTC drops too, but NOT below the floor — the case + // a reconnect or SET_CLOCK can still resolve. + final wandering = RecordGate(); + expect(wandering.admit(wall + 10 * 86400, wallNow: wall), isFalse); + expect(wandering.dropped, 1); + expect(wandering.droppedBelowFloor, 0); + expect(wandering.timeBaseNotWallClock, isFalse); + + // One below-floor drop among others is not a verdict about the source. + final mixed = RecordGate(); + expect(mixed.admit(1000000000, wallNow: wall), isFalse); + expect(mixed.admit(wall + 10 * 86400, wallNow: wall), isFalse); + expect(mixed.timeBaseNotWallClock, isFalse); + + // Never a verdict on a gate that has rejected nothing. + expect(RecordGate().timeBaseNotWallClock, isFalse); + }); + test('frontier seed from the durable cursor is honoured', () { final g = RecordGate(frontierTs: wall - 50); expect(g.frontierTs, wall - 50); From b38427e251b70784308dff0787327f333b7063dd Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:38:53 +0530 Subject: [PATCH 03/23] ios: the picker reads the plist instead of a second copy of the uuids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AccessorySetup.swift carried its own hardcoded pair of whoop service uuids alongside the ones in Info.plist. apple already requires every descriptor criterion to be declared there, so that array is by definition the complete set — the swift constants are just a second copy that can drift. deleted them; showPicker builds its items from Bundle.main now, so adding a band is a dart-only edit. the plist array is generated from the registry by tool/, and a unit test asserts the committed file matches. a script phase that rewrites a tracked file mid-build fails open — stale plist, build still green — which is the failure mode worth avoiding here. four .first calls made ios look single-device when it isn't. two were real bugs, not just truncation: the picker completion returned accessories.first, so once a second band is provisioned it hands dart the OLD device's uuid to connect to. and dropping the restored peripherals lost the arc retain on ones still carrying a pending connect that bluetoothd holds — a peripheral we don't hold is one cancelPending can't cancel, so the two centrals fight over it. note showPicker still can't add a second band: BleRestoreManager.start makes a central at launch whenever a band is paired, and the picker fails once any central exists. the addAnother path is plumbing, and the teardown belongs with the device screen. left the early return in so a repeat pair tap doesn't become a guaranteed failure. also corrected the TN3115 claim in the header — note 5 attaches to force-quit and the control-centre toggle only, and it's apps GAINING relaunch, not losing it. the real stakes are that on ios 18+ the picker is the pairing path, so a missing uuid means the band can't be paired at all. --- ios/Runner/AccessorySetup.swift | 125 +++++++++++++++++++---------- ios/Runner/BleRestoreManager.swift | 26 +++--- ios/Runner/Info.plist | 14 ++++ test/ios_ask_plist_test.dart | 44 ++++++++++ tool/gen_ios_ask_plist.dart | 97 ++++++++++++++++++++++ 5 files changed, 254 insertions(+), 52 deletions(-) create mode 100644 test/ios_ask_plist_test.dart create mode 100644 tool/gen_ios_ask_plist.dart diff --git a/ios/Runner/AccessorySetup.swift b/ios/Runner/AccessorySetup.swift index ee7c0a86..c68a4a14 100644 --- a/ios/Runner/AccessorySetup.swift +++ b/ios/Runner/AccessorySetup.swift @@ -7,11 +7,19 @@ import AccessorySetupKit /// AccessorySetupKit (ASK) bridge — iOS 18+ only. /// -/// WHY: per Apple TN3115, starting in iOS 26 the OS only relaunches a *terminated* app -/// into the background for a Bluetooth accessory that was provisioned via ASK. Our -/// CoreBluetooth state-restoration central (BleRestoreManager) still does the actual -/// relaunch/pending-connect work, but iOS 26 will only honour it if the peripheral was -/// set up through the ASK picker. So pairing on iOS 18+ goes through this picker. +/// WHY: TN3115's relaunch table has two rows that are "No" without ASK and "Yes" with it — +/// "App Force Quit by the user" and "Control Center Bluetooth button toggled". Note 5: +/// "Starting in iOS 26 and iPadOS 26, only apps that use AccessorySetupKit to setup +/// Bluetooth accessories will be relaunched." Apple DTS has since clarified on the forums +/// that note 5 scopes to *those two rows only* — it is a capability apps GAIN, not one they +/// lose, and the ordinary relaunch cases (app removed from memory, crashed, device +/// restarted) never depended on ASK. The condition is also stated per-APP, not +/// per-accessory; the relaunch itself still fires "if and only if" a pending Core Bluetooth +/// request completes, which is what BleRestoreManager holds. +/// +/// So the load-bearing reason to route pairing through the picker is simpler than +/// "otherwise no background sync": on iOS 18+ this picker IS how a user grants us the +/// accessory, plus those two extra relaunch cases. /// /// COEXISTENCE: ASK is a provisioning/authorization gate, NOT a connection owner. It hands /// back `ASAccessory.bluetoothIdentifier` — the CoreBluetooth peripheral UUID, which is the @@ -21,18 +29,17 @@ import AccessorySetupKit /// /// Dart MethodChannel `openstrap/accessory_setup`: /// - `isSupported` -> Bool (true only on iOS 18+) -/// - `provisionedId` -> String?(uppercased UUID of an already-provisioned WHOOP, or nil) -/// - `showPicker` -> String (the provisioned band's UUID; throws on cancel/error) +/// - `provisionedId` -> String?(uppercased UUID of an already-provisioned band, or nil) +/// - `showPicker` -> String (the band provisioned by THIS call; throws on cancel/error) +/// optional Bool argument: true = add another accessory /// - `removeAll` -> nil (deprovision all — used on unpair) +/// +/// The service UUIDs the picker matches on are NOT duplicated here: they come from +/// Info.plist's NSAccessorySetupBluetoothServices, which Apple requires to list every +/// descriptor criterion anyway, and which is generated from `kBandRegistry` by +/// `tool/gen_ios_ask_plist.dart`. enum AccessorySetup { private static let channelName = "openstrap/accessory_setup" - // WHOOP GATT service UUIDs, one per generation (match GattProfile in Dart). - // `fileprivate` so the iOS-18 Impl below can read them. BOTH must also be - // listed in Info.plist under NSAccessorySetupBluetoothServices. - // • gen4 ("Harvard", WHOOP 4) — 6108… - // • gen5 ("fd4b", WHOOP 5) — fd4b… (EXPERIMENTAL) - fileprivate static let whoopServiceUUIDGen4 = "61080001-8d6d-82b8-614a-1c8cb0f8dcc6" - fileprivate static let whoopServiceUUIDGen5 = "fd4b0001-cce1-4033-93ce-002d5875f58a" static func register(messenger: FlutterBinaryMessenger) { let channel = FlutterMethodChannel(name: channelName, binaryMessenger: messenger) @@ -43,14 +50,19 @@ enum AccessorySetup { case "provisionedId": if #available(iOS 18.0, *) { - Impl.shared.provisionedId { result($0) } + // ponytail: the channel still hands Dart ONE id because the Dart side is + // still two SharedPreferences scalars (change-list E3). Swift holds the + // whole array; widen this to a list when the device table lands. + Impl.shared.provisionedIds { result($0.first) } } else { result(nil) } case "showPicker": if #available(iOS 18.0, *) { - Impl.shared.showPicker { res in + // No argument (today's only caller) = today's behaviour exactly. + let addAnother = (call.arguments as? Bool) ?? false + Impl.shared.showPicker(addAnother: addAnother) { res in switch res { case .success(let id): result(id) case .failure(let err): @@ -114,28 +126,44 @@ private final class Impl { } } - /// Returns the uppercased UUID of an already-provisioned WHOOP, or nil. - func provisionedId(_ completion: @escaping (String?) -> Void) { + /// Every provisioned accessory's uppercased CoreBluetooth UUID, in session order. + /// `session.accessories` is an ARRAY — one entry per accessory the user has granted. + private var provisionedIdList: [String] { + session.accessories.compactMap { $0.bluetoothIdentifier?.uuidString.uppercased() } + } + + /// Returns the uppercased UUIDs of the already-provisioned accessories (possibly empty). + func provisionedIds(_ completion: @escaping ([String]) -> Void) { ensureActivated() // `accessories` is reliable only after activation has reported .activated; give the // session a brief beat to populate on a cold start, then read it. queue.asyncAfter(deadline: .now() + 0.2) { [weak self] in - guard let self = self else { completion(nil); return } - let id = self.session.accessories - .compactMap { $0.bluetoothIdentifier } - .first? - .uuidString - .uppercased() - completion(id) + guard let self = self else { completion([]); return } + completion(self.provisionedIdList) } } - func showPicker(_ completion: @escaping (Result) -> Void) { + /// - Parameter addAnother: show the picker even though an accessory is already + /// provisioned, i.e. provision a SECOND band. See the ordering warning below. + func showPicker(addAnother: Bool = false, + _ completion: @escaping (Result) -> Void) { ensureActivated() - // Already provisioned? Don't re-show the picker — just return the known id. - if let existing = session.accessories - .compactMap({ $0.bluetoothIdentifier }) - .first?.uuidString.uppercased() { + let known = provisionedIdList + // Already provisioned and not explicitly adding another? Don't re-show the picker — + // just return the known id. + // + // ORDERING (do not "fix" this into an unconditional showPicker): ASK's picker fails + // with "CBManager is active with global permissions" once ANY CBCentralManager exists + // in the process, and BleRestoreManager creates one at launch on every already-paired + // launch. So this early return is also what keeps a repeat "pair" tap from turning + // into a guaranteed picker failure. + // + // ponytail: `addAnother` is therefore plumbing, not a working second-band flow — a + // second accessory can only be provisioned while no central is alive (fresh install, + // or after unpair, which releases the restore central via BleRestoreManager.disarm). + // A real "add a band" flow has to tear both centrals down first; that belongs with + // the device table (change-list E3/E4), not here. + if !addAnother, let existing = known.first { completion(.success(existing)) return } @@ -149,10 +177,15 @@ private final class Impl { // require an NSAccessorySetupBluetoothNames entry and risk excluding the band // on a name mismatch.) // - // ASK matches ANY item in the picker list, so we offer one item per WHOOP - // generation: gen4 (WHOOP 4) and gen5 (WHOOP 5, experimental). A band that - // advertises either service can be provisioned; the provisioned identifier is - // the same CoreBluetooth UUID regardless of generation. + // ASK matches ANY item in the picker list, so we offer one item per band in the + // registry. A band advertising any listed service can be provisioned; the + // provisioned identifier is the same CoreBluetooth UUID whichever it is. + // + // The list comes straight from Info.plist rather than a Swift copy: Apple requires + // every descriptor criterion to be declared there, so that array is by definition + // the complete set — a second copy here could only ever be the stale one. It is + // generated from kBandRegistry (tool/gen_ios_ask_plist.dart, pinned by + // test/ios_ask_plist_test.dart), so adding a band stays a one-file edit in Dart. let productImage = UIImage(named: "StrapProduct") ?? UIImage(systemName: "sensor.tag.radiowave.forward") ?? UIImage() @@ -162,10 +195,15 @@ private final class Impl { return ASPickerDisplayItem( name: name, productImage: productImage, descriptor: descriptor) } - let items = [ - item(AccessorySetup.whoopServiceUUIDGen4, "WHOOP band"), - item(AccessorySetup.whoopServiceUUIDGen5, "WHOOP 5 band"), - ] + let info = Bundle.main.infoDictionary ?? [:] + let services = info["NSAccessorySetupBluetoothServices"] as? [String] ?? [] + let labels = info["OSBandLabels"] as? [String: String] ?? [:] + let items = services.map { item($0, labels[$0.uppercased()] ?? "Band") } + guard !items.isEmpty else { + completion(.failure(PickerError( + message: "No accessory services are declared in Info.plist."))) + return + } pickerResult = completion session.showPicker(for: items) { [weak self] error in @@ -177,10 +215,13 @@ private final class Impl { } return } - // Picker succeeded — read the newly provisioned accessory's identifier. - let id = self.session.accessories - .compactMap { $0.bluetoothIdentifier } - .first?.uuidString.uppercased() + // Picker succeeded — return the accessory THIS run added, not `accessories.first`: + // once a second band is provisioned the first entry is the OLD one, so `.first` + // would hand Dart the wrong device to connect to. (With none previously known — + // every pairing today — the added one IS the first, so this is unchanged.) + let current = self.provisionedIdList + let knownSet = Set(known) + let id = current.first { !knownSet.contains($0) } ?? current.first if let cb = self.pickerResult { self.pickerResult = nil if let id = id { diff --git a/ios/Runner/BleRestoreManager.swift b/ios/Runner/BleRestoreManager.swift index 7634ae10..82db4233 100644 --- a/ios/Runner/BleRestoreManager.swift +++ b/ios/Runner/BleRestoreManager.swift @@ -47,7 +47,12 @@ class BleRestoreManager: NSObject { private var central: CBCentralManager? private var bandUUID: UUID? - private var pending: CBPeripheral? // retained so ARC doesn't drop it mid-connect + // Peripherals we hold a pending connect for, retained so ARC doesn't drop them + // mid-connect. An array because willRestoreState hands back an array: dropping all but + // the first loses the retain on the rest, and a peripheral we no longer hold is one + // cancelPending can no longer cancel — so the two centrals would fight over it when the + // app reclaims the band. With one provisioned band there is exactly one entry. + private var pending: [CBPeripheral] = [] private var channel: FlutterMethodChannel? private var flutterReady = false private var wakeQueuedBeforeReady = false @@ -239,14 +244,14 @@ class BleRestoreManager: NSObject { NSLog("[ble-restore] band not retrievable yet") return } - pending = p + pending = [p] central.connect(p, options: nil) // no timeout → persists, relaunches us when reachable NSLog("[ble-restore] armed pending connect") } private func cancelPending() { - if let p = pending { central?.cancelPeripheralConnection(p) } - pending = nil + for p in pending { central?.cancelPeripheralConnection(p) } + pending = [] } private func disarm() { @@ -327,12 +332,13 @@ extension BleRestoreManager: CBCentralManagerDelegate { } func centralManager(_ central: CBCentralManager, willRestoreState dict: [String: Any]) { - if let restored = dict[CBCentralManagerRestoredStatePeripheralsKey] as? [CBPeripheral], - let p = restored.first { - pending = p - NSLog("[ble-restore] willRestoreState restored \(restored.count) peripheral(s)") - // The pending/active connect was preserved; didConnect fires if it lands. - } + let restored = dict[CBCentralManagerRestoredStatePeripheralsKey] as? [CBPeripheral] ?? [] + guard !restored.isEmpty else { return } + // Take ALL of them, not just the first — the count was already being logged, so the + // code always knew there could be more. Each carries a pending/active connect + // bluetoothd preserved for us; didConnect fires per peripheral if one lands. + pending = restored + NSLog("[ble-restore] willRestoreState restored \(restored.count) peripheral(s)") } func centralManager(_ central: CBCentralManager, didConnect peripheral: CBPeripheral) { diff --git a/ios/Runner/Info.plist b/ios/Runner/Info.plist index 338d3723..bf050401 100644 --- a/ios/Runner/Info.plist +++ b/ios/Runner/Info.plist @@ -51,6 +51,11 @@ $(FLUTTER_BUILD_NUMBER) LSRequiresIPhoneOS + NSAccessorySetupBluetoothServices 61080001-8D6D-82B8-614A-1C8CB0F8DCC6 @@ -62,6 +67,15 @@ NSBluetoothAlwaysUsageDescription OpenStrap connects to your WHOOP band over Bluetooth to sync your health data. + + OSBandLabels + + 61080001-8D6D-82B8-614A-1C8CB0F8DCC6 + WHOOP 4 + FD4B0001-CCE1-4033-93CE-002D5875F58A + WHOOP 5 + \n' + '\t\t$kFd4bMemberUuid16\n'; String _labelsBody(List registry) => registry .map((e) => '\t\t${e.service.toUpperCase()}\n' From a327f4effffb98b022586d55ee95a1cbcda0c0e8 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:26:00 +0530 Subject: [PATCH 23/23] tool: use a StringBuffer instead of + string concat in the plist generator flutter analyze flagged prefer_interpolation_to_compose_strings on the FD4B fallback block added in the merge fix. Output is byte-identical (dart run tool/gen_ios_ask_plist.dart confirms Info.plist unchanged). --- tool/gen_ios_ask_plist.dart | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tool/gen_ios_ask_plist.dart b/tool/gen_ios_ask_plist.dart index 3462893d..165f6c8d 100644 --- a/tool/gen_ios_ask_plist.dart +++ b/tool/gen_ios_ask_plist.dart @@ -54,17 +54,22 @@ String _esc(String s) => s .replaceAll('<', '<') .replaceAll('>', '>'); -String _servicesBody(List registry) => registry - .map((e) => '\t\t${e.service.toUpperCase()}\n') - .join() + - '\t\t\n' - '\t\t$kFd4bMemberUuid16\n'; +String _servicesBody(List registry) { + final buf = StringBuffer(); + for (final e in registry) { + buf.writeln('\t\t${e.service.toUpperCase()}'); + } + buf + ..writeln('\t\t') + ..writeln('\t\t$kFd4bMemberUuid16'); + return buf.toString(); +} String _labelsBody(List registry) => registry .map((e) => '\t\t${e.service.toUpperCase()}\n'